01:27:07.378 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:27:07.392 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task9_3e26/test-tmp
01:27:07.395 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:27:07.400 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task9_3e26/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:27:07.702 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:27:07.901 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:27:07.902 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:27:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:07.906582 374 mini_oidc.cc:152] Starting JWKS server
01:27:07.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:07.911106 374 webserver.cc:460] Webserver started at http://127.0.0.1:44203/ using document root <none> and password file <none>
01:27:07.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:07.911791 374 mini_oidc.cc:189] Starting OIDC Discovery server
01:27:07.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:07.914671 374 webserver.cc:460] Webserver started at http://127.0.0.1:44093/ using document root <none> and password file <none>
01:27:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:27:07Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:27:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:27:07Z Disabled control of system clock
01:27:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:07.973383 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/wal
01:27:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/data
01:27:07.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:07.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:07.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:07.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:07.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/logs
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/data/info.pb
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:35733
01:27:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
01:27:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:07.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44203/jwks.json with env {}
01:27:08.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.315460 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:08.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.316049 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:08.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.316812 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.351723 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.352042 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.352246 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.352424 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.352648 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.352855 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44203/jwks.json
01:27:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:08.391445 386 master_runner.cc:386] Master server non-default flags:
01:27:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:08.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:08.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:08.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/data
01:27:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/wal
01:27:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:35733
01:27:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44203/jwks.json
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/data/info.pb
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/logs
01:27:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:08.392921 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:08.395426 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.421248 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.421526 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:08.422757 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.570050 386 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:27:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:09.570843 394 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:27:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.577021 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:09.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.580448 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:09.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.581923 386 hybrid_clock.cc:648] HybridClock initialized: now 1781918829581845 us; error 93 us; skew 500 ppm
01:27:09.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.583056 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:09.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.591591 386 webserver.cc:460] Webserver started at http://127.0.93.190:37079/ using document root <none> and password file <none>
01:27:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.592763 386 fs_manager.cc:359] Metadata directory not provided
01:27:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.593008 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:09.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.594033 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:09.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.599584 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/data/instance:
01:27:09.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abdd127ac8144d30b792c764cfe0bb7a"
01:27:09.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:09 on dist-test-slave-wg14"
01:27:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.601105 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/wal/instance:
01:27:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abdd127ac8144d30b792c764cfe0bb7a"
01:27:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:09 on dist-test-slave-wg14"
01:27:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.611672 386 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.005s
01:27:09.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.619867 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.621323 386 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
01:27:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.621747 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/wal
01:27:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abdd127ac8144d30b792c764cfe0bb7a"
01:27:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:09 on dist-test-slave-wg14"
01:27:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.622201 386 fs_report.cc:383] FS layout report
01:27:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/wal
01:27:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/wal
01:27:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/data/data
01:27:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.660398 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:09.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.662113 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.662698 386 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.755182 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:35733
01:27:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.755378 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:35733 every 8 connection(s)
01:27:09.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.758255 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/data/info.pb
01:27:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.765549 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 386
01:27:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.765974 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-0/wal/instance
01:27:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.766054 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:27:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.769836 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/wal
01:27:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/data
01:27:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/logs
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/data/info.pb
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:09.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:40161
01:27:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
01:27:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:09.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:09.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44203/jwks.json with env {}
01:27:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.783625 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } has no permanent_uuid. Determining permanent_uuid...
01:27:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.815631 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } has no permanent_uuid. Determining permanent_uuid...
01:27:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:09.818975 405 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40161: connect: Connection refused (error 111)
01:27:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:09.824450 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40161: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40161: connect: Connection refused (error 111)
01:27:09.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.874202 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } attempt: 1
01:27:09.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:09.880275 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40161: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40161: connect: Connection refused (error 111)
01:27:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:09.948748 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } attempt: 2
01:27:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:09.952533 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40161: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40161: connect: Connection refused (error 111)
01:27:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:10.043980 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } attempt: 3
01:27:10.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.048781 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40161: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40161: connect: Connection refused (error 111)
01:27:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.116212 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.116796 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.117301 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.148973 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.149274 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.149471 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:10.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.149645 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:10.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.149879 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:10.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.150116 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44203/jwks.json
01:27:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:10.187657 456 master_runner.cc:386] Master server non-default flags:
01:27:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/data
01:27:10.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/wal
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:40161
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44203/jwks.json
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/data/info.pb
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/logs
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:10.188865 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:10.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:10.191182 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:10.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:10.192255 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } attempt: 4
01:27:10.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.195828 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40161: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40161: connect: Connection refused (error 111)
01:27:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.214334 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:10.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:10.495430 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } attempt: 5
01:27:10.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.503577 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40161: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40161: connect: Connection refused (error 111)
01:27:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.051298 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } attempt: 6
01:27:11.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.060550 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40161: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40161: connect: Connection refused (error 111)
01:27:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.215415 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:10.216233 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:10.218111 456 server_base.cc:1034] running on GCE node
01:27:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.376097 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.378798 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.380328 456 hybrid_clock.cc:648] HybridClock initialized: now 1781918831380267 us; error 51 us; skew 500 ppm
01:27:11.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.381739 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.388692 456 webserver.cc:460] Webserver started at http://127.0.93.189:40739/ using document root <none> and password file <none>
01:27:11.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.389822 456 fs_manager.cc:359] Metadata directory not provided
01:27:11.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.390060 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:11.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.390599 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.396698 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/data/instance:
01:27:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3"
01:27:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:11 on dist-test-slave-wg14"
01:27:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.398411 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/wal/instance:
01:27:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3"
01:27:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:11 on dist-test-slave-wg14"
01:27:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.407557 456 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
01:27:11.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.414894 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.416893 456 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
01:27:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.417371 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/wal
01:27:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3"
01:27:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:11 on dist-test-slave-wg14"
01:27:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.417874 456 fs_report.cc:383] FS layout report
01:27:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/wal
01:27:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/wal
01:27:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/data/data
01:27:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:11.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.469334 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:11.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.470975 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:11.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.471664 456 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.567770 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:40161
01:27:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.567857 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:40161 every 8 connection(s)
01:27:11.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.570756 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/data/info.pb
01:27:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.577663 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:27:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.580950 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 456
01:27:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.581426 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-1/wal/instance
01:27:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.584528 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/wal
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/data
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/logs
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/data/info.pb
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:44873
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44203/jwks.json with env {}
01:27:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.591894 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } has no permanent_uuid. Determining permanent_uuid...
01:27:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.627758 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } has no permanent_uuid. Determining permanent_uuid...
01:27:11.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.655277 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } has no permanent_uuid. Determining permanent_uuid...
01:27:11.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.659427 479 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.666105 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.715852 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 1
01:27:11.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.721421 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.789927 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 2
01:27:11.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.794883 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:11.886508 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 3
01:27:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.891384 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:11.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.978308 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:11.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.979038 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:11.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:11.979741 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:12.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.016153 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:12.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.016512 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:12.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.016861 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:12.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.017196 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:12.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.017526 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.017746 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44203/jwks.json
01:27:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.034897 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 4
01:27:12.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.040607 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.060029 531 master_runner.cc:386] Master server non-default flags:
01:27:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/data
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/wal
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:44873
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44203/jwks.json
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/data/info.pb
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/logs
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:12.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:12.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.061676 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:12.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.065191 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:12.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.097787 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:12.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.121263 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } attempt: 7
01:27:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.139810 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } has no permanent_uuid. Determining permanent_uuid...
01:27:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.147580 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.206177 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 1
01:27:12.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.211041 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:12.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.292645 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 2
01:27:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.297219 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:12.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.340724 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 5
01:27:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.345791 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:12.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.382661 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 3
01:27:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.387398 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.527851 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 4
01:27:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.532831 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.816284 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 5
01:27:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.821353 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.893252 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 6
01:27:12.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.903110 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.098356 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:12.099651 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:12.104226 531 server_base.cc:1034] running on GCE node
01:27:13.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.320013 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:13.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.323513 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:13.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.324976 531 hybrid_clock.cc:648] HybridClock initialized: now 1781918833324936 us; error 56 us; skew 500 ppm
01:27:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.326089 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:13.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.338909 531 webserver.cc:460] Webserver started at http://127.0.93.188:42845/ using document root <none> and password file <none>
01:27:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.340132 531 fs_manager.cc:359] Metadata directory not provided
01:27:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.340390 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.341122 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.346433 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/data/instance:
01:27:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "899b2c7c462a496c8dbd1c46b8d92701"
01:27:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:13 on dist-test-slave-wg14"
01:27:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.347973 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/wal/instance:
01:27:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "899b2c7c462a496c8dbd1c46b8d92701"
01:27:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:13 on dist-test-slave-wg14"
01:27:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.357003 531 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
01:27:13.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.363566 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.364799 531 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
01:27:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.365139 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/wal
01:27:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "899b2c7c462a496c8dbd1c46b8d92701"
01:27:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:13 on dist-test-slave-wg14"
01:27:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.365463 531 fs_report.cc:383] FS layout report
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/wal
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/wal
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/data/data
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.373787 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 6
01:27:13.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:13.380466 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:44873: Network error: Client connection negotiation failed: client connection to 127.0.93.188:44873: connect: Connection refused (error 111)
01:27:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.409003 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.410497 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:13.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.410940 531 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.500177 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:44873
01:27:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.500326 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:44873 every 8 connection(s)
01:27:13.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.504096 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/data/info.pb
01:27:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.505028 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 531
01:27:13.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.505456 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/master-2/wal/instance
01:27:13.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.512564 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:27:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.535014 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } has no permanent_uuid. Determining permanent_uuid...
01:27:13.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.567592 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } has no permanent_uuid. Determining permanent_uuid...
01:27:13.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.584774 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } has no permanent_uuid. Determining permanent_uuid...
01:27:13.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.607427 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701: Bootstrap starting.
01:27:13.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.614003 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701: Neither blocks nor log segments found. Creating new log.
01:27:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.616313 609 log.cc:824] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701: Log is configured to *not* fsync() on all Append() calls
01:27:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.623588 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701: No bootstrap required, opened a new log
01:27:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.649245 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.649924 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:27:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.650183 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 899b2c7c462a496c8dbd1c46b8d92701, State: Initialized, Role: FOLLOWER
01:27:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.651095 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.653707 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.654584 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: This master's current role is: FOLLOWER
01:27:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.655965 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: configured and running, proceeding with master startup.
01:27:13.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:13.666801 563 tablet.cc:2369] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:27:13.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:13.692255 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:13.693040 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.686540 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:35733: OK, instance_id { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" instance_seqno: 1781918829729389 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:27:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.715596 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:27:13.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.743304 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:27:13.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.744103 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:13.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.746351 618 leader_election.cc:288] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abdd127ac8144d30b792c764cfe0bb7a (127.0.93.190:35733), a9d3dd2e8e834c0c914fc4e3f6d2f8d3 (127.0.93.189:40161)
01:27:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.963768 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 7
01:27:13.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:13.994074 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: Bootstrap starting.
01:27:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.001075 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: Neither blocks nor log segments found. Creating new log.
01:27:14.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.004549 529 log.cc:824] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: Log is configured to *not* fsync() on all Append() calls
01:27:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.010644 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: No bootstrap required, opened a new log
01:27:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.047552 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.048302 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:27:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.048578 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a9d3dd2e8e834c0c914fc4e3f6d2f8d3, State: Initialized, Role: FOLLOWER
01:27:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.049713 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:14.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.053238 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.054646 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [sys.catalog]: This master's current role is: FOLLOWER
01:27:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.055349 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [sys.catalog]: configured and running, proceeding with master startup.
01:27:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.056175 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "899b2c7c462a496c8dbd1c46b8d92701" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" is_pre_election: true
01:27:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.058696 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 899b2c7c462a496c8dbd1c46b8d92701 in term 0.
01:27:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.060722 560 leader_election.cc:302] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 899b2c7c462a496c8dbd1c46b8d92701, a9d3dd2e8e834c0c914fc4e3f6d2f8d3; no voters:
01:27:14.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.062031 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:27:14.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.062399 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:27:14.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.062745 618 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 0 FOLLOWER]: Advancing to term 1
01:27:14.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.071380 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:14.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.075565 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "899b2c7c462a496c8dbd1c46b8d92701" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3"
01:27:14.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.076323 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [term 0 FOLLOWER]: Advancing to term 1
01:27:14.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.093343 618 leader_election.cc:288] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [CANDIDATE]: Term 1 election: Requested vote from peers abdd127ac8144d30b792c764cfe0bb7a (127.0.93.190:35733), a9d3dd2e8e834c0c914fc4e3f6d2f8d3 (127.0.93.189:40161)
01:27:14.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.101444 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 899b2c7c462a496c8dbd1c46b8d92701 in term 1.
01:27:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.103752 560 leader_election.cc:302] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 899b2c7c462a496c8dbd1c46b8d92701, a9d3dd2e8e834c0c914fc4e3f6d2f8d3; no voters:
01:27:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.105830 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 1 FOLLOWER]: Leader election won for term 1
01:27:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.109057 618 raft_consensus.cc:686] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [term 1 LEADER]: Becoming Leader. State: Replica: 899b2c7c462a496c8dbd1c46b8d92701, State: Running, Role: LEADER
01:27:14.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.112888 618 consensus_queue.cc:237] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:14.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.142688 642 sys_catalog.cc:455] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 899b2c7c462a496c8dbd1c46b8d92701. Latest consensus state: current_term: 1 leader_uuid: "899b2c7c462a496c8dbd1c46b8d92701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:14.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.144096 642 sys_catalog.cc:458] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: This master's current role is: LEADER
01:27:14.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.151563 646 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:27:14.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.174387 646 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:27:14.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:14.197594 647 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:14.198704 647 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:14.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.223706 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [term 1 FOLLOWER]: Refusing update from remote peer 899b2c7c462a496c8dbd1c46b8d92701: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:27:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.226438 642 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:27:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.161877 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:35733: OK, instance_id { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" instance_seqno: 1781918829729389 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:27:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.231870 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:27:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:14.232093 483 tablet.cc:2369] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:27:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.279156 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 899b2c7c462a496c8dbd1c46b8d92701. Latest consensus state: current_term: 1 leader_uuid: "899b2c7c462a496c8dbd1c46b8d92701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.280977 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [sys.catalog]: This master's current role is: FOLLOWER
01:27:14.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.297729 650 mvcc.cc:205] Tried to move back new op lower bound from 7298739544951586816 to 7298739544555778048. Current Snapshot: MvccSnapshot[applied={T|T < 7298739544951586816}]
01:27:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.320317 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "899b2c7c462a496c8dbd1c46b8d92701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.321251 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: This master's current role is: LEADER
01:27:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.331151 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "899b2c7c462a496c8dbd1c46b8d92701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.331938 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3 [sys.catalog]: This master's current role is: FOLLOWER
01:27:14.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.339516 646 catalog_manager.cc:1347] Generated new cluster ID: e671d1c1fa564ab486e6d5b964856687
01:27:14.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.339874 646 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:27:14.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.392607 646 catalog_manager.cc:1370] Generated new certificate authority record
01:27:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.397818 646 catalog_manager.cc:1504] Loading token signing keys...
01:27:14.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.413926 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } attempt: 7
01:27:14.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.456439 646 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701: Generated new TSK 0
01:27:14.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.457821 646 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:27:14.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.464797 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a: Bootstrap starting.
01:27:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.484504 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a: Neither blocks nor log segments found. Creating new log.
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.498394 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/wal
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/data
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/logs
01:27:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/data/info.pb
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:27:14.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.501170 454 log.cc:824] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a: Log is configured to *not* fsync() on all Append() calls
01:27:14.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.518824 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a: No bootstrap required, opened a new log
01:27:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.569607 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.571381 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:27:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.572417 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: abdd127ac8144d30b792c764cfe0bb7a, State: Initialized, Role: FOLLOWER
01:27:14.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.574433 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } }
01:27:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.604103 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.605013 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [sys.catalog]: This master's current role is: FOLLOWER
01:27:14.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.606762 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [sys.catalog]: configured and running, proceeding with master startup.
01:27:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.606946 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "899b2c7c462a496c8dbd1c46b8d92701" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "abdd127ac8144d30b792c764cfe0bb7a" is_pre_election: true
01:27:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.607370 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "899b2c7c462a496c8dbd1c46b8d92701" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "abdd127ac8144d30b792c764cfe0bb7a"
01:27:14.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.608809 427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [term 0 FOLLOWER]: Advancing to term 1
01:27:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.616460 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [term 1 FOLLOWER]: Refusing update from remote peer 899b2c7c462a496c8dbd1c46b8d92701: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:27:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.619326 618 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [LEADER]: Connected to new peer: Peer: permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:27:14.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:14.676023 408 tablet.cc:2369] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:27:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.750526 669 sys_catalog.cc:455] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "899b2c7c462a496c8dbd1c46b8d92701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.755120 669 sys_catalog.cc:458] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [sys.catalog]: This master's current role is: FOLLOWER
01:27:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.755707 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 899b2c7c462a496c8dbd1c46b8d92701. Latest consensus state: current_term: 1 leader_uuid: "899b2c7c462a496c8dbd1c46b8d92701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.756480 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a [sys.catalog]: This master's current role is: FOLLOWER
01:27:14.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.811496 673 catalog_manager.cc:1259] Loaded cluster ID: e671d1c1fa564ab486e6d5b964856687
01:27:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.812362 673 catalog_manager.cc:1552] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a: loading cluster ID for follower catalog manager: success
01:27:14.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.834010 673 catalog_manager.cc:1574] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a: acquiring CA information for follower catalog manager: success
01:27:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:14.844847 673 catalog_manager.cc:1602] T 00000000000000000000000000000000 P abdd127ac8144d30b792c764cfe0bb7a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:27:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.156144 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "899b2c7c462a496c8dbd1c46b8d92701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abdd127ac8144d30b792c764cfe0bb7a" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35733 } } peers { permanent_uuid: "a9d3dd2e8e834c0c914fc4e3f6d2f8d3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40161 } } peers { permanent_uuid: "899b2c7c462a496c8dbd1c46b8d92701" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 44873 } } }
01:27:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.157159 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P 899b2c7c462a496c8dbd1c46b8d92701 [sys.catalog]: This master's current role is: LEADER
01:27:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.206250 647 catalog_manager.cc:1259] Loaded cluster ID: e671d1c1fa564ab486e6d5b964856687
01:27:15.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.206970 647 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: loading cluster ID for follower catalog manager: success
01:27:15.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.216104 647 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: acquiring CA information for follower catalog manager: success
01:27:15.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.243718 647 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a9d3dd2e8e834c0c914fc4e3f6d2f8d3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:27:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:15.365376 656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:15.366088 656 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:15.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:15.366977 656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:15.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:15.435966 656 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:15.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:15.437444 656 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
01:27:15.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.509125 656 tablet_server_runner.cc:78] Tablet server non-default flags:
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/data
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/wal
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/data/info.pb
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/logs
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.511340 656 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:15.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:15.514284 656 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:15.576792 683 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:18.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:16.949564 682 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 656
01:27:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:18.767926 682 kernel_stack_watchdog.cc:198] Thread 656 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 343ms:
01:27:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:27:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:27:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:27:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:27:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:15.608369 684 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:18.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:18.772888 686 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:18.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:18.771636 656 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 3.168s user 0.838s sys 1.379s
01:27:18.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:18.773840 656 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 3.171s user 0.839s sys 1.379s
01:27:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:18.775784 685 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:27:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.776491 656 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:27:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.778946 656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:18.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.782249 656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:18.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.784250 656 hybrid_clock.cc:648] HybridClock initialized: now 1781918838784173 us; error 68 us; skew 500 ppm
01:27:18.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.786167 656 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:18.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.799520 656 webserver.cc:460] Webserver started at http://127.0.93.129:36349/ using document root <none> and password file <none>
01:27:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.800901 656 fs_manager.cc:359] Metadata directory not provided
01:27:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.801215 656 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:18.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.801771 656 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.807567 656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/data/instance:
01:27:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "241a2a805f0c4f1bb9f48ffad952426d"
01:27:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:18 on dist-test-slave-wg14"
01:27:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.809373 656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/wal/instance:
01:27:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "241a2a805f0c4f1bb9f48ffad952426d"
01:27:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:18 on dist-test-slave-wg14"
01:27:18.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.817967 656 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.005s
01:27:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.824294 695 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.825712 656 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
01:27:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.826128 656 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/wal
01:27:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "241a2a805f0c4f1bb9f48ffad952426d"
01:27:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:18 on dist-test-slave-wg14"
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.826552 656 fs_report.cc:383] FS layout report
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/wal
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/wal
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/data/data
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:18.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.879850 656 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.881615 656 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.882169 656 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.885133 656 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:27:18.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.890184 656 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:27:18.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.890415 656 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:27:18.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.890712 656 ts_tablet_manager.cc:610] Registered 0 tablets
01:27:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:18.890913 656 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:27:19.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.065114 656 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:43809
01:27:19.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.065243 807 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:43809 every 8 connection(s)
01:27:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.068418 656 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/data/info.pb
01:27:19.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.072953 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 656
01:27:19.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.073673 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-0/wal/instance
01:27:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.087044 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/wal
01:27:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/data
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/logs
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/data/info.pb
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:27:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.133929 808 heartbeater.cc:347] Connected to a master server at 127.0.93.188:44873
01:27:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.134467 808 heartbeater.cc:464] Registering TS with master...
01:27:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.135828 808 heartbeater.cc:511] Master 127.0.93.188:44873 requested a full tablet report, sending...
01:27:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.139715 574 ts_manager.cc:194] Registered new tserver with Master: 241a2a805f0c4f1bb9f48ffad952426d (127.0.93.129:43809)
01:27:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.140378 809 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40161
01:27:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.140767 809 heartbeater.cc:464] Registering TS with master...
01:27:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.141782 809 heartbeater.cc:511] Master 127.0.93.189:40161 requested a full tablet report, sending...
01:27:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.142709 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:39959
01:27:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.144834 812 heartbeater.cc:347] Connected to a master server at 127.0.93.190:35733
01:27:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.145219 812 heartbeater.cc:464] Registering TS with master...
01:27:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.145367 494 ts_manager.cc:194] Registered new tserver with Master: 241a2a805f0c4f1bb9f48ffad952426d (127.0.93.129:43809)
01:27:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.146559 812 heartbeater.cc:511] Master 127.0.93.190:35733 requested a full tablet report, sending...
01:27:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.150218 419 ts_manager.cc:194] Registered new tserver with Master: 241a2a805f0c4f1bb9f48ffad952426d (127.0.93.129:43809)
01:27:19.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:19.472622 818 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:19.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:19.473212 818 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:19.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:19.473836 818 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:19.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:19.506927 818 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:19.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:19.508042 818 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.544991 818 tablet_server_runner.cc:78] Tablet server non-default flags:
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/data
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/wal
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/data/info.pb
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/logs
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:27:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.546567 818 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:19.548811 818 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:19.581280 827 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.149751 808 heartbeater.cc:503] Master 127.0.93.188:44873 was elected leader, sending a full tablet report...
01:27:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:19.584187 824 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:19.582170 825 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:20.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:20.799599 826 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1219 milliseconds
01:27:20.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.799778 818 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:27:20.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.801048 818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:20.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.803731 818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:20.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.805413 818 hybrid_clock.cc:648] HybridClock initialized: now 1781918840805360 us; error 59 us; skew 500 ppm
01:27:20.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.806274 818 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:20.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.813241 818 webserver.cc:460] Webserver started at http://127.0.93.130:41553/ using document root <none> and password file <none>
01:27:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.814179 818 fs_manager.cc:359] Metadata directory not provided
01:27:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.814378 818 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.814769 818 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:20.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.819876 818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/data/instance:
01:27:20.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60dbe941e57348438a79e13ffc2aac4d"
01:27:20.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:20 on dist-test-slave-wg14"
01:27:20.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.821120 818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/wal/instance:
01:27:20.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60dbe941e57348438a79e13ffc2aac4d"
01:27:20.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:20 on dist-test-slave-wg14"
01:27:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.829192 818 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
01:27:20.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.835389 837 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:20.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.837149 818 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
01:27:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.837548 818 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/wal
01:27:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60dbe941e57348438a79e13ffc2aac4d"
01:27:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:20 on dist-test-slave-wg14"
01:27:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.837993 818 fs_report.cc:383] FS layout report
01:27:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/wal
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/wal
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/data/data
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.877193 818 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.878880 818 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.879472 818 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:20.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.882305 818 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:27:20.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.887485 818 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:27:20.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.887732 818 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:27:20.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.888041 818 ts_tablet_manager.cc:610] Registered 0 tablets
01:27:20.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:20.888229 818 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:27:21.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.112252 818 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:41211
01:27:21.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.112380 949 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:41211 every 8 connection(s)
01:27:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.115572 818 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/data/info.pb
01:27:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.119369 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 818
01:27:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.119938 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-1/wal/instance
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.176415 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/wal
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/data
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/logs
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/data/info.pb
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:27:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:27:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.180508 950 heartbeater.cc:347] Connected to a master server at 127.0.93.188:44873
01:27:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.181731 950 heartbeater.cc:464] Registering TS with master...
01:27:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.183254 950 heartbeater.cc:511] Master 127.0.93.188:44873 requested a full tablet report, sending...
01:27:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.184456 951 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40161
01:27:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.184808 951 heartbeater.cc:464] Registering TS with master...
01:27:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.185638 951 heartbeater.cc:511] Master 127.0.93.189:40161 requested a full tablet report, sending...
01:27:21.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.186560 574 ts_manager.cc:194] Registered new tserver with Master: 60dbe941e57348438a79e13ffc2aac4d (127.0.93.130:41211)
01:27:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.188711 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:43415
01:27:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.189241 494 ts_manager.cc:194] Registered new tserver with Master: 60dbe941e57348438a79e13ffc2aac4d (127.0.93.130:41211)
01:27:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.215495 952 heartbeater.cc:347] Connected to a master server at 127.0.93.190:35733
01:27:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.215922 952 heartbeater.cc:464] Registering TS with master...
01:27:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.216737 952 heartbeater.cc:511] Master 127.0.93.190:35733 requested a full tablet report, sending...
01:27:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.219695 419 ts_manager.cc:194] Registered new tserver with Master: 60dbe941e57348438a79e13ffc2aac4d (127.0.93.130:41211)
01:27:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:21.585042 959 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:21.585605 959 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:21.586153 959 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:21.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:21.619756 959 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:21.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:21.620620 959 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
01:27:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.658156 959 tablet_server_runner.cc:78] Tablet server non-default flags:
01:27:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:40823
01:27:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/data
01:27:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/wal
01:27:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/data/info.pb
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35733,127.0.93.189:40161,127.0.93.188:44873
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/logs
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:21.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:21.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.659650 959 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:21.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.661868 959 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:21.686158 965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:22.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.196909 950 heartbeater.cc:503] Master 127.0.93.188:44873 was elected leader, sending a full tablet report...
01:27:22.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:21.686662 966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:22.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:21.687839 968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:22.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:21.689229 959 server_base.cc:1034] running on GCE node
01:27:22.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.852751 959 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.855692 959 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.857890 959 hybrid_clock.cc:648] HybridClock initialized: now 1781918842857819 us; error 69 us; skew 500 ppm
01:27:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.859114 959 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:22.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.866276 959 webserver.cc:460] Webserver started at http://127.0.93.131:41481/ using document root <none> and password file <none>
01:27:22.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.867470 959 fs_manager.cc:359] Metadata directory not provided
01:27:22.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.867723 959 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:22.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.868275 959 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:22.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.873538 959 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/data/instance:
01:27:22.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd00c9cd246b4706a9297d52a8cb7713"
01:27:22.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:22 on dist-test-slave-wg14"
01:27:22.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.874809 959 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/wal/instance:
01:27:22.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd00c9cd246b4706a9297d52a8cb7713"
01:27:22.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:22 on dist-test-slave-wg14"
01:27:22.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.884521 959 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.000s
01:27:22.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.892478 976 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:22.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.894021 959 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
01:27:22.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.894430 959 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/wal
01:27:22.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd00c9cd246b4706a9297d52a8cb7713"
01:27:22.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:22 on dist-test-slave-wg14"
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.894843 959 fs_report.cc:383] FS layout report
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/wal
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/wal
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/data/data
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.950999 959 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.952656 959 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:22.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.953275 959 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.956523 959 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:27:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.962397 959 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:27:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.962862 959 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
01:27:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.963382 959 ts_tablet_manager.cc:610] Registered 0 tablets
01:27:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:22.963609 959 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:27:23.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.174220 959 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:39279
01:27:23.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.174342 1088 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:39279 every 8 connection(s)
01:27:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.177294 959 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/data/info.pb
01:27:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.187732 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 959
01:27:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.188345 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster8445419034606532453/ts-2/wal/instance
01:27:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.241217 1091 heartbeater.cc:347] Connected to a master server at 127.0.93.190:35733
01:27:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.241937 1091 heartbeater.cc:464] Registering TS with master...
01:27:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.243391 1090 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40161
01:27:23.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.243974 1090 heartbeater.cc:464] Registering TS with master...
01:27:23.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.244086 1091 heartbeater.cc:511] Master 127.0.93.190:35733 requested a full tablet report, sending...
01:27:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.245114 1090 heartbeater.cc:511] Master 127.0.93.189:40161 requested a full tablet report, sending...
01:27:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.246716 1089 heartbeater.cc:347] Connected to a master server at 127.0.93.188:44873
01:27:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.247174 1089 heartbeater.cc:464] Registering TS with master...
01:27:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.248438 1089 heartbeater.cc:511] Master 127.0.93.188:44873 requested a full tablet report, sending...
01:27:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.248349 494 ts_manager.cc:194] Registered new tserver with Master: bd00c9cd246b4706a9297d52a8cb7713 (127.0.93.131:39279)
01:27:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.248438 419 ts_manager.cc:194] Registered new tserver with Master: bd00c9cd246b4706a9297d52a8cb7713 (127.0.93.131:39279)
01:27:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.250975 574 ts_manager.cc:194] Registered new tserver with Master: bd00c9cd246b4706a9297d52a8cb7713 (127.0.93.131:39279)
01:27:23.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.253163 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:39497
01:27:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:23.272162 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:27:23.286 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:27:23.288 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:27:23.299 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 35733
}
bound_http_address {
host: "127.0.93.190"
port: 37079
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 40161
}
bound_http_address {
host: "127.0.93.189"
port: 40739
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 44873
}
bound_http_address {
host: "127.0.93.188"
port: 42845
}
}
}
01:27:23.304 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:27:23.308 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 43809
}
bound_http_address {
host: "127.0.93.129"
port: 36349
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 41211
}
bound_http_address {
host: "127.0.93.130"
port: 41553
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 39279
}
bound_http_address {
host: "127.0.93.131"
port: 41481
}
}
}
01:27:23.308 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:27:23.519 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:27:23.666 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task9_3e26/test-tmp
01:27:23.667 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:27:23.667 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task9_3e26/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:27:23.671 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:27:24.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:24.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:27:24.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:24.063001 1100 mini_oidc.cc:152] Starting JWKS server
01:27:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:24.067703 1100 webserver.cc:460] Webserver started at http://127.0.0.1:33365/ using document root <none> and password file <none>
01:27:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:24.068352 1100 mini_oidc.cc:189] Starting OIDC Discovery server
01:27:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:24.071098 1100 webserver.cc:460] Webserver started at http://127.0.0.1:42475/ using document root <none> and password file <none>
01:27:24.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:27:24.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/principal' for realm 'KRBTEST.COM',
01:27:24.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:27:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:24 dist-test-slave-wg14 krb5kdc[1112](info): setting up network...
01:27:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:27:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:24 dist-test-slave-wg14 krb5kdc[1112](info): set up 2 sockets
01:27:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:24 dist-test-slave-wg14 krb5kdc[1112](info): commencing operation
01:27:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:27:24.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:24.258843 1089 heartbeater.cc:503] Master 127.0.93.188:44873 was elected leader, sending a full tablet report...
01:27:25.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.166584 1100 mini_kdc.cc:121] Time spent starting KDC: real 1.064s user 0.010s sys 0.016s
01:27:25.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:27:25.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:27:25.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:27:25.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:27:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:27:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:27:25.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:27:25.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:27:25.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:27:25.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:27:25.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/test-user.keytab.
01:27:25.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/test-user.keytab.
01:27:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:25 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918845, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:27:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:27:25Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:27:25.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:27:25Z Disabled control of system clock
01:27:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.62@KRBTEST.COM; defaulting to no policy
01:27:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:27:25.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.62@KRBTEST.COM" created.
01:27:25.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:27:25.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.62.keytab.
01:27:25.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.62.keytab.
01:27:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:25.384235 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data
01:27:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/logs
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/info.pb
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:46769
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.62.keytab
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:27:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.722854 1126 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:25.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.723544 1126 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.724159 1126 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.758984 1126 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:27:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.759289 1126 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.759579 1126 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.759783 1126 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:25.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.759974 1126 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:25.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.760164 1126 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:25.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.760305 1126 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33365/jwks.json
01:27:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:25.799436 1126 master_runner.cc:386] Master server non-default flags:
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.62.keytab
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:46769
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/info.pb
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/logs
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:25.800706 1126 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:25.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:25.801841 1126 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:25.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.824431 1133 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.826911 1132 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:25.827693 1126 server_base.cc:1034] running on GCE node
01:27:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:25.824499 1135 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:27.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.030609 1126 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.033715 1126 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.035266 1126 hybrid_clock.cc:648] HybridClock initialized: now 1781918847035201 us; error 65 us; skew 500 ppm
01:27:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.036154 1126 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:27.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:27 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918847, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:27.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.043458 1126 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
01:27:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.050468 1126 webserver.cc:460] Webserver started at http://127.1.19.62:32985/ using document root <none> and password file <none>
01:27:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.051519 1126 fs_manager.cc:359] Metadata directory not provided
01:27:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.051733 1126 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.052142 1126 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:27.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.058808 1126 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/instance:
01:27:27.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5edb09270444211922ae5dcc10b3be2"
01:27:27.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:27 on dist-test-slave-wg14"
01:27:27.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.060343 1126 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal/instance:
01:27:27.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5edb09270444211922ae5dcc10b3be2"
01:27:27.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:27 on dist-test-slave-wg14"
01:27:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.070400 1126 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.005s
01:27:27.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.077723 1143 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:27.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.079088 1126 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.079553 1126 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5edb09270444211922ae5dcc10b3be2"
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:27 on dist-test-slave-wg14"
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.080000 1126 fs_report.cc:383] FS layout report
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/data
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:27.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:27.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.125905 1126 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:27.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.132622 1126 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:27.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.133096 1126 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.239346 1126 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:46769
01:27:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.239439 1194 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:46769 every 8 connection(s)
01:27:27.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.242915 1126 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/info.pb
01:27:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.250391 1195 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:27:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.253211 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 1126
01:27:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.253715 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal/instance
01:27:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.266975 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } has no permanent_uuid. Determining permanent_uuid...
01:27:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.61@KRBTEST.COM; defaulting to no policy
01:27:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:27:27.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.61@KRBTEST.COM" created.
01:27:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:27 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918847, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
01:27:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:27:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.61.keytab.
01:27:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.61.keytab.
01:27:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.308506 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/logs
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/info.pb
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:34117
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.61.keytab
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
01:27:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:27:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.320760 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } has no permanent_uuid. Determining permanent_uuid...
01:27:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.323362 1145 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.329107 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:34117: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.378728 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } attempt: 1
01:27:27.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.382408 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:34117: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:27.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.451195 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } attempt: 2
01:27:27.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.456663 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:34117: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.548247 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } attempt: 3
01:27:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.553138 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:34117: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.687983 1201 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.688735 1201 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.689445 1201 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.696583 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } attempt: 4
01:27:27.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.701395 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:34117: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.724843 1201 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:27:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.725240 1201 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.725646 1201 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.725956 1201 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.726248 1201 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.726531 1201 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.726740 1201 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33365/jwks.json
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.774066 1201 master_runner.cc:386] Master server non-default flags:
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:27.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.61.keytab
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:34117
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/info.pb
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/logs
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.775777 1201 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.777357 1201 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:27.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.800907 1208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:28.000929 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } attempt: 5
01:27:28.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:28.005228 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:34117: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:28.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:28.552996 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } attempt: 6
01:27:28.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:28.561700 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:34117: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:27.803694 1201 server_base.cc:1034] running on GCE node
01:27:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.801316 1211 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:27.801301 1209 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:29.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.005206 1201 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.008152 1201 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.009605 1201 hybrid_clock.cc:648] HybridClock initialized: now 1781918849009572 us; error 45 us; skew 500 ppm
01:27:29.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.010612 1201 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:29.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:29 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918849, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:29.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.017802 1201 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
01:27:29.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.024606 1201 webserver.cc:460] Webserver started at http://127.1.19.61:45499/ using document root <none> and password file <none>
01:27:29.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.025871 1201 fs_manager.cc:359] Metadata directory not provided
01:27:29.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.026198 1201 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:29.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.026805 1201 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:29.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.031931 1201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/instance:
01:27:29.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea767071649f45fab000280c6b143939"
01:27:29.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:29 on dist-test-slave-wg14"
01:27:29.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.033151 1201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal/instance:
01:27:29.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea767071649f45fab000280c6b143939"
01:27:29.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:29 on dist-test-slave-wg14"
01:27:29.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.041962 1201 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
01:27:29.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.048946 1223 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:29.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.050477 1201 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
01:27:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.050839 1201 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea767071649f45fab000280c6b143939"
01:27:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:29 on dist-test-slave-wg14"
01:27:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.051266 1201 fs_report.cc:383] FS layout report
01:27:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/data
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.092939 1201 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:29.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.102265 1201 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.102754 1201 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.208710 1201 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:34117
01:27:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.209324 1274 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:34117 every 8 connection(s)
01:27:29.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.213230 1201 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/info.pb
01:27:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.215991 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 1201
01:27:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.216579 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal/instance
01:27:29.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.220968 1275 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:27:29.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.60@KRBTEST.COM; defaulting to no policy
01:27:29.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.239915 1275 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } has no permanent_uuid. Determining permanent_uuid...
01:27:29.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:27:29.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.60@KRBTEST.COM" created.
01:27:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:27:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.60.keytab.
01:27:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.60.keytab.
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.278271 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/logs
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/info.pb
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:33973
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.60.keytab
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:27:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:27:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:27:29.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:29 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918849, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
01:27:29.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.302944 1275 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } has no permanent_uuid. Determining permanent_uuid...
01:27:29.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:29 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918849, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
01:27:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.334657 1275 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } has no permanent_uuid. Determining permanent_uuid...
01:27:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.337858 1225 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.343626 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.393328 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 1
01:27:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.398066 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.466570 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 2
01:27:29.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.470820 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.562295 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 3
01:27:29.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.566732 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.622275 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } attempt: 7
01:27:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.631855 1281 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.632594 1281 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.633893 1281 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:29.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:29 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918847, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
01:27:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.654402 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } has no permanent_uuid. Determining permanent_uuid...
01:27:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.661834 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.673359 1281 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:27:29.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.673631 1281 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:29.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.673908 1281 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:29.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.674108 1281 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:29.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.674290 1281 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:29.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.674472 1281 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:29.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.674602 1281 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33365/jwks.json
01:27:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.710212 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 4
01:27:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.714082 1281 master_runner.cc:386] Master server non-default flags:
01:27:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.60.keytab
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:33973
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/info.pb
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/logs
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.714656 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.715533 1281 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.716653 1281 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:29.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.720283 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 1
01:27:29.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.724977 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.738875 1292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:29.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.806493 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 2
01:27:29.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.811986 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:29.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.897575 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 3
01:27:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.902902 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:30.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:30.014165 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 5
01:27:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:30.019443 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:30.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:30.043354 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 4
01:27:30.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:30.049482 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:30.332953 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 5
01:27:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:30.339502 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:30.567153 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 6
01:27:30.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:30.575780 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:30.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:30.891954 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } attempt: 6
01:27:30.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:30.899708 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:33973: Network error: Client connection negotiation failed: client connection to 127.1.19.60:33973: connect: Connection refused (error 111)
01:27:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.740648 1293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:29.741200 1295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:29.741760 1281 server_base.cc:1034] running on GCE node
01:27:31.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.017514 1281 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:31.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.020623 1281 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:31.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.022185 1281 hybrid_clock.cc:648] HybridClock initialized: now 1781918851022138 us; error 65 us; skew 500 ppm
01:27:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.023253 1281 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:31.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918851, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.031311 1281 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
01:27:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.038722 1281 webserver.cc:460] Webserver started at http://127.1.19.60:38813/ using document root <none> and password file <none>
01:27:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.040061 1281 fs_manager.cc:359] Metadata directory not provided
01:27:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.040315 1281 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:31.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.041081 1281 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:27:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.047616 1281 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/instance:
01:27:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfb241e1d7bf41f3b63602831cdccfcd"
01:27:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:31 on dist-test-slave-wg14"
01:27:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.049278 1281 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal/instance:
01:27:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfb241e1d7bf41f3b63602831cdccfcd"
01:27:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:31 on dist-test-slave-wg14"
01:27:31.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.059015 1281 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
01:27:31.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.066553 1308 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:31.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.068662 1281 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.006s sys 0.000s
01:27:31.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.069128 1281 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:31.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfb241e1d7bf41f3b63602831cdccfcd"
01:27:31.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:31 on dist-test-slave-wg14"
01:27:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.069567 1281 fs_report.cc:383] FS layout report
01:27:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/data
01:27:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:31.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.114485 1281 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:31.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.121013 1281 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:31.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.121479 1281 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:31.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.235376 1281 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:33973
01:27:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.235455 1359 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:33973 every 8 connection(s)
01:27:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.242588 1281 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/info.pb
01:27:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.250972 1360 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:27:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.252691 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 1281
01:27:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.253116 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal/instance
01:27:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.269827 1360 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } has no permanent_uuid. Determining permanent_uuid...
01:27:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918845, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
01:27:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918851, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
01:27:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.315217 1360 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } has no permanent_uuid. Determining permanent_uuid...
01:27:31.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918851, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
01:27:31.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918845, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
01:27:31.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.347244 1360 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } has no permanent_uuid. Determining permanent_uuid...
01:27:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918851, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
01:27:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918845, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
01:27:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.383006 1360 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: Bootstrap starting.
01:27:31.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.390830 1100 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:27:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.391773 1360 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: Neither blocks nor log segments found. Creating new log.
01:27:31.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.394323 1360 log.cc:824] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: Log is configured to *not* fsync() on all Append() calls
01:27:31.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.399967 1360 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: No bootstrap required, opened a new log
01:27:31.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:31.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:27:31.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:27:31.420 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:27:31.424 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.62"
port: 46769
}
bound_http_address {
host: "127.1.19.62"
port: 32985
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.61"
port: 34117
}
bound_http_address {
host: "127.1.19.61"
port: 45499
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.60"
port: 33973
}
bound_http_address {
host: "127.1.19.60"
port: 38813
}
}
}
01:27:31.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:27:31.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:31.426 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:27:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.425961 1360 raft_consensus.cc:348] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.426625 1360 raft_consensus.cc:374] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:27:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.426884 1360 raft_consensus.cc:729] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bfb241e1d7bf41f3b63602831cdccfcd, State: Initialized, Role: FOLLOWER
01:27:31.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:27:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.427989 1360 consensus_queue.cc:260] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.431813 1369 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } } }
01:27:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.432807 1369 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [sys.catalog]: This master's current role is: FOLLOWER
01:27:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.433861 1360 sys_catalog.cc:564] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [sys.catalog]: configured and running, proceeding with master startup.
01:27:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:27:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:31.470755 1382 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:31.471252 1382 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:31.471 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:31.474 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc
01:27:31.481 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:27:31.487 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc
01:27:31.489 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:27:31.495 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
01:27:31.497 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:27:31.497 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:27:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.467396 1281 master_runner.cc:186] Error getting master registration for 127.1.19.62:46769: OK, instance_id { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" instance_seqno: 1781918847212148 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:27:31.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.508255 1281 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:27:31.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:31 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:31.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:27:31.539 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:31.539 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc
01:27:31.541 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:27:31.542 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.61:34117
01:27:31.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:27:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.544890 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 1201
01:27:31.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:31.571 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.60:33973
01:27:31.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:27:31.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.572944 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 1281
01:27:31.623 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:31.623 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.62:46769
01:27:31.624 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:27:31.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.625433 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 1126
01:27:31.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:31.647 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.61:34117
01:27:31.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:31.651553 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/logs
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/info.pb
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:34117
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45499
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.61.keytab
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:27:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.011776 1387 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.012481 1387 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.013268 1387 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.047992 1387 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:27:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.048285 1387 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.048718 1387 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:32.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.049042 1387 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:32.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.049350 1387 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:32.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.049641 1387 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:32.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.049855 1387 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33365/jwks.json
01:27:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:32.091037 1387 master_runner.cc:386] Master server non-default flags:
01:27:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data
01:27:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.61.keytab
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:34117
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/info.pb
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45499
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/logs
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:32.093117 1387 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:32.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:32.094779 1387 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.115727 1398 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.116833 1395 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:32.116847 1396 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.367280 1397 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1248 milliseconds
01:27:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.367422 1387 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:27:33.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.369289 1387 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:33.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.372979 1387 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.374752 1387 hybrid_clock.cc:648] HybridClock initialized: now 1781918853374707 us; error 67 us; skew 500 ppm
01:27:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.375881 1387 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:33.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:33 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918853, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:33.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.382658 1387 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
01:27:33.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.389767 1387 webserver.cc:460] Webserver started at http://127.1.19.61:45499/ using document root <none> and password file <none>
01:27:33.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.390949 1387 fs_manager.cc:359] Metadata directory not provided
01:27:33.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.391284 1387 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:33.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.401041 1387 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.000s sys 0.004s
01:27:33.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.409313 1408 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:33.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.412084 1387 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.002s sys 0.004s
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.412635 1387 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea767071649f45fab000280c6b143939"
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:29 on dist-test-slave-wg14"
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.416082 1387 fs_report.cc:383] FS layout report
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/data
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:33.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.456725 1387 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:33.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.463292 1387 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.464010 1387 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:33.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.554747 1387 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:34117
01:27:33.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.554853 1459 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:34117 every 8 connection(s)
01:27:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.558130 1387 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/data/info.pb
01:27:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.565689 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 1387
01:27:33.566 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:33.566 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.60:33973
01:27:33.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:27:33.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.568984 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:33.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:33.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:33.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/logs
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/info.pb
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:33973
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38813
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.60.keytab
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:27:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:33.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:27:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.574561 1460 sys_catalog.cc:263] Verifying existing consensus state
01:27:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 01:27:33.576404 1460 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:27:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.576951 1387 master.cc:537] Master@127.1.19.61:34117 shutting down...
01:27:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:33.641484 1387 master.cc:552] Master@127.1.19.61:34117 shutdown complete.
01:27:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:27:33.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.936550 1462 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:33.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.937183 1462 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:33.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.937784 1462 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:33.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.982218 1462 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:27:33.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.983220 1462 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.983970 1462 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.984385 1462 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.984683 1462 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:33.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.985126 1462 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:33.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:33.985657 1462 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33365/jwks.json
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:34.035969 1462 master_runner.cc:386] Master server non-default flags:
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:27:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.60.keytab
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:33973
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/info.pb
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38813
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:34.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/logs
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:34.037647 1462 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:34.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:34.040014 1462 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:34.062448 1469 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:34.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:34.321182 1085 debug-util.cc:398] Leaking SignalData structure 0x7b08000af2c0 after lost signal to thread 960
01:27:34.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:34.322657 1085 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd60 after lost signal to thread 1088
01:27:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:34.065526 1462 server_base.cc:1034] running on GCE node
01:27:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:34.062491 1470 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:34.062848 1472 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:35.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.313964 1462 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:35.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.317656 1462 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:35.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.319350 1462 hybrid_clock.cc:648] HybridClock initialized: now 1781918855319112 us; error 217 us; skew 500 ppm
01:27:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.321902 1462 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:35 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918855, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.336400 1462 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
01:27:35.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.346966 1462 webserver.cc:460] Webserver started at http://127.1.19.60:38813/ using document root <none> and password file <none>
01:27:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.348778 1462 fs_manager.cc:359] Metadata directory not provided
01:27:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.349123 1462 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.361809 1462 fs_manager.cc:709] Time spent opening directory manager: real 0.008s user 0.005s sys 0.004s
01:27:35.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.368954 1482 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.370927 1462 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
01:27:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.371572 1462 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfb241e1d7bf41f3b63602831cdccfcd"
01:27:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:31 on dist-test-slave-wg14"
01:27:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.375150 1462 fs_report.cc:383] FS layout report
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/wal
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/data
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:35.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.423522 1462 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.429930 1462 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.430450 1462 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:35.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.524328 1462 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:33973
01:27:35.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.524509 1533 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:33973 every 8 connection(s)
01:27:35.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.528623 1462 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-2/data/info.pb
01:27:35.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:35.533 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.62:46769
01:27:35.533 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:27:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.530695 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 1462
01:27:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.536346 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9_3e26/build/tsan/bin/kudu
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/logs
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/info.pb
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:27:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:46769
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32985
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.62.keytab
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:27:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.556471 1534 sys_catalog.cc:263] Verifying existing consensus state
01:27:35.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.565665 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: Bootstrap starting.
01:27:35.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.611048 1534 log.cc:824] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: Log is configured to *not* fsync() on all Append() calls
01:27:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.617101 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:27:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.618017 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: Bootstrap complete.
01:27:35.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.656114 1534 raft_consensus.cc:348] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:35.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.659826 1534 raft_consensus.cc:374] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:27:35.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.660154 1534 raft_consensus.cc:729] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bfb241e1d7bf41f3b63602831cdccfcd, State: Initialized, Role: FOLLOWER
01:27:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.661191 1534 consensus_queue.cc:260] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.664464 1538 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } } }
01:27:35.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.666224 1538 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [sys.catalog]: This master's current role is: FOLLOWER
01:27:35.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.666706 1534 sys_catalog.cc:564] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [sys.catalog]: configured and running, proceeding with master startup.
01:27:35.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:35.687731 1488 tablet.cc:2369] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:27:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:35.715317 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:35.715677 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.715507 1462 master_runner.cc:186] Error getting master registration for 127.1.19.62:46769: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111),
01:27:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.749758 1462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:27:35.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.885222 1538 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:27:35.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.885810 1538 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:35.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:35.891665 1485 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:27:35.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.896764 1538 leader_election.cc:288] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769), ea767071649f45fab000280c6b143939 (127.1.19.61:34117)
01:27:35.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:35.909181 1484 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea767071649f45fab000280c6b143939 (127.1.19.61:34117): Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:35.911983 1485 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769): Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:27:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.912438 1485 leader_election.cc:302] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bfb241e1d7bf41f3b63602831cdccfcd; no voters: b5edb09270444211922ae5dcc10b3be2, ea767071649f45fab000280c6b143939
01:27:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:35.913326 1538 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:27:36.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.078230 1536 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:27:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.079439 1536 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:27:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.080673 1536 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:27:36.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.127521 1536 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:27:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.127839 1536 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:27:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.128345 1536 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:27:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.128743 1536 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:27:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.129074 1536 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:27:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.129386 1536 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:27:36.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.129633 1536 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33365/jwks.json
01:27:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:36.177106 1536 master_runner.cc:386] Master server non-default flags:
01:27:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:27:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:37269
01:27:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:27:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:27:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data
01:27:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:46769,127.1.19.61:34117,127.1.19.60:33973
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/krb5kdc/kudu_127.1.19.62.keytab
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:46769
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33365/jwks.json
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/info.pb
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32985
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:27:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/logs
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 00:16:57 UTC on ccycloud.quasar-agmwty.root.comops.site
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456770
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:27:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:36.178907 1536 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:27:36.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:36.180181 1536 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:27:36.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.207233 1558 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.720531 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.720885 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:37.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.209372 1559 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:37.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:36.221537 1561 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:27:37.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:37.682662 1560 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1473 milliseconds
01:27:37.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.682848 1536 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:27:37.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.686623 1536 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:27:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.690033 1536 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:27:37.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.691541 1536 hybrid_clock.cc:648] HybridClock initialized: now 1781918857691461 us; error 91 us; skew 500 ppm
01:27:37.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.693037 1536 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:27:37.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:37 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918857, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.700230 1536 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
01:27:37.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.706166 1536 webserver.cc:460] Webserver started at http://127.1.19.62:32985/ using document root <none> and password file <none>
01:27:37.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.707247 1536 fs_manager.cc:359] Metadata directory not provided
01:27:37.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.707499 1536 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:27:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.717666 1536 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.003s
01:27:37.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.722858 1569 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:27:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.724217 1536 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:27:37.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:37.725445 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:37.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:37.725920 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.724678 1536 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data,/tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5edb09270444211922ae5dcc10b3be2"
01:27:37.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 01:27:27 on dist-test-slave-wg14"
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.727722 1536 fs_report.cc:383] FS layout report
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/data
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:27:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:27:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:27:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:27:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:27:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:27:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:27:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:27:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.775692 1536 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:27:37.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.783627 1536 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:27:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.784421 1536 kserver.cc:163] Server-wide thread pool size limit: 3276
01:27:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.862466 1610 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:27:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.863082 1610 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:37.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.865327 1610 leader_election.cc:288] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769), ea767071649f45fab000280c6b143939 (127.1.19.61:34117)
01:27:37.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:37.905148 1485 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769): Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:27:37.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:37.905814 1484 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea767071649f45fab000280c6b143939 (127.1.19.61:34117): Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:37.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.906296 1484 leader_election.cc:302] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bfb241e1d7bf41f3b63602831cdccfcd; no voters: b5edb09270444211922ae5dcc10b3be2, ea767071649f45fab000280c6b143939
01:27:37.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.907241 1610 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:27:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.915587 1536 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:46769
01:27:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.915715 1625 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:46769 every 8 connection(s)
01:27:37.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.918849 1536 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/data/info.pb
01:27:37.921 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:27:37.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.920646 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9_3e26/build/tsan/bin/kudu as pid 1536
01:27:37.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.937065 1626 sys_catalog.cc:263] Verifying existing consensus state
01:27:37.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 01:27:37.938313 1626 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:27:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:37.939334 1536 master.cc:537] Master@127.1.19.62:46769 shutting down...
01:27:38.007 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:38.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:38.041199 1536 master.cc:552] Master@127.1.19.62:46769 shutdown complete.
01:27:38.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9_3e26/test-tmp/mini-kudu-cluster3181994450313106590/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:27:38.131 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.60:33973(127.1.19.60:33973)] connecting to peer
01:27:38.142 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:38.181 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.19.60:33973(127.1.19.60:33973)] Successfully connected to peer
01:27:38.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.193 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.200 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.203 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.205 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.318 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
01:27:38.329 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.60
01:27:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:38 dist-test-slave-wg14 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
01:27:38.634 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2c4a37d5, L:/127.0.0.1:36188 - R:/127.1.19.60:33973] using SASL/GSSAPI
01:27:38.658 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:38.674 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:38.680 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.680 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:38.680 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.681 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.685 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.691 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:38.713 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:38.715 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:38.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.721 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:38.729038 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:38.729338 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:38.733 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:38.734 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:38.737 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.737 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.738 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.738 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.741 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.743 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:38.753 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:38.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:38.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.763 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.763 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.764 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.764 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:38.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:38.775 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:38.776 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.777 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.777 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:38.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:38.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:38.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.858 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.862 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:38.913 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:38.917 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:38.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.921 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.921 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.922 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:38.922 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:38.924 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:39.112 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:39.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:39.123 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.124 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.125 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.127 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.128 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.129 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:39.232 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:39.234 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:39.237 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.237 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.237 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.240 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.242 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:39.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:39.732362 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:39.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:39.732684 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:39.765846 1635 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:27:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:39.766250 1635 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:39.768113 1635 leader_election.cc:288] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769), ea767071649f45fab000280c6b143939 (127.1.19.61:34117)
01:27:39.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:39.774132 1485 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769): Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:27:39.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:39.775573 1484 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea767071649f45fab000280c6b143939 (127.1.19.61:34117): Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:39.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:39.775985 1484 leader_election.cc:302] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bfb241e1d7bf41f3b63602831cdccfcd; no voters: b5edb09270444211922ae5dcc10b3be2, ea767071649f45fab000280c6b143939
01:27:39.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:39.776815 1635 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:27:39.933 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:39.934 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:39.936 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.936 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.937 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:39.937 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.937 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:39.942 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:40.073 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:40.074 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:40.076 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:40.076 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:40.076 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:40.078 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:40.078 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:40.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:40.082 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:40.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:40.735782 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:40.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:40.736060 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:40.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:40.752887 1462 master_runner.cc:186] Error getting master registration for 127.1.19.62:46769: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111),
01:27:40.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:40.753265 1462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:27:41.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:41.739187 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:41.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:41.739483 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:42.742568 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:42.742854 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:43.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:43.745792 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:43.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:43.746133 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:43.953 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:43.954 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:43.957 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:43.957 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:43.957 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:43.958 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:43.958 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:43.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:43.961 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:44.192 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:44.194 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:44.197 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:44.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:44.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:44.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:44.199 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:44.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:44.202 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:44.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:44.553128 1645 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:27:44.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:44.553978 1645 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:44.556882 1645 leader_election.cc:288] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769), ea767071649f45fab000280c6b143939 (127.1.19.61:34117)
01:27:44.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:44.560931 1485 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111) [suppressed 5 similar messages]
01:27:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:44.566248 1485 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769): Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:27:44.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:44.567232 1484 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea767071649f45fab000280c6b143939 (127.1.19.61:34117): Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:44.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:44.567660 1484 leader_election.cc:302] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bfb241e1d7bf41f3b63602831cdccfcd; no voters: b5edb09270444211922ae5dcc10b3be2, ea767071649f45fab000280c6b143939
01:27:44.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:44.568439 1645 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:27:44.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:44.750140 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:44.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:44.750566 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:45.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:45 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918865, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:45.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:45.343758 1478 init.cc:295] Successfully reacquired a new kerberos TGT
01:27:45.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:45.755334 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:45.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:45.755729 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:45.758847 1462 master_runner.cc:186] Error getting master registration for 127.1.19.62:46769: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111),
01:27:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:45.759725 1462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:27:46.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:46.699704 1651 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:27:46.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:46.700083 1651 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:46.701663 1651 leader_election.cc:288] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769), ea767071649f45fab000280c6b143939 (127.1.19.61:34117)
01:27:46.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:46.706408 1485 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769): Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:27:46.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:46.706951 1484 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea767071649f45fab000280c6b143939 (127.1.19.61:34117): Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:46.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:46.707358 1484 leader_election.cc:302] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bfb241e1d7bf41f3b63602831cdccfcd; no voters: b5edb09270444211922ae5dcc10b3be2, ea767071649f45fab000280c6b143939
01:27:46.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:46.707929 1651 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:27:46.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:46.759311 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:46.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:46.759618 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:47.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:47.762945 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:47.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:47.763393 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:47.833 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:47.834 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:47.836 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:47.837 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:47.837 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:47.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:47.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:47.838 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:47.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:48.766592 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:48.766942 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:49.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:49.770032 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:49.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:49.770335 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:50.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:50.764237 1462 master_runner.cc:186] Error getting master registration for 127.1.19.62:46769: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111),
01:27:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:50.764778 1462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:27:50.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:50.773362 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:50.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:50.773680 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:51.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:51.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:51.316 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:51.316 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:51.317 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:51.317 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:51.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:51.318 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:51.320 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:51.713 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:51.714 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:51.716 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:51.716 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:51.716 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:51.717 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:51.717 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:51.717 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:51.718 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:51.777334 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:51.777683 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:52.780728 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:52.781733 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:53.785362 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:53.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:53.785816 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:53.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:53.820689 1662 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:27:53.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:53.821064 1662 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:27:53.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:53.822783 1662 leader_election.cc:288] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769), ea767071649f45fab000280c6b143939 (127.1.19.61:34117)
01:27:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:53.825075 1484 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111) [suppressed 3 similar messages]
01:27:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:53.827788 1485 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769): Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:27:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:53.828244 1484 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea767071649f45fab000280c6b143939 (127.1.19.61:34117): Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:27:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:53.828598 1484 leader_election.cc:302] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bfb241e1d7bf41f3b63602831cdccfcd; no voters: b5edb09270444211922ae5dcc10b3be2, ea767071649f45fab000280c6b143939
01:27:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:53.829188 1662 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:27:54.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:54.789726 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:54.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:54.790109 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:27:55 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918875, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:27:55.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:55.346387 1478 init.cc:295] Successfully reacquired a new kerberos TGT
01:27:55.472 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:55.474 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:55.476 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:55.476 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:55.477 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:55.477 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:55.477 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:55.477 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:55.479 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:55.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:55.769236 1462 master_runner.cc:186] Error getting master registration for 127.1.19.62:46769: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111),
01:27:55.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:27:55.769780 1462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:27:55.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:55.793545 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:55.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:55.793855 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:56.796722 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:56.796983 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:57.012 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:57.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:57.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:57.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:57.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:57.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:57.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:57.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:57.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:57.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:57.799962 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:57.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:57.800256 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:58.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:58.339529 450 debug-util.cc:398] Leaking SignalData structure 0x7b08000a2400 after lost signal to thread 387
01:27:58.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:58.340582 450 debug-util.cc:398] Leaking SignalData structure 0x7b08000a04c0 after lost signal to thread 453
01:27:58.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:58.803073 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:58.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:58.803472 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:27:59.012 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:27:59.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:27:59.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:59.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:59.018 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:27:59.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:59.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:59.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:27:59.023 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:27:59.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:59.806602 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:27:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:27:59.806903 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:28:00.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:28:00.294 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:28:00.296 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:00.296 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:00.296 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:00.296 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:00.296 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:00.296 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:00.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:28:00.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:00.772541 1462 master_runner.cc:186] Error getting master registration for 127.1.19.62:46769: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111),
01:28:00.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:00.772882 1462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:28:00.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:00.809914 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:28:00.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:00.810284 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:28:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:00.898603 1676 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:28:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:00.899302 1676 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:28:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:00.901490 1676 leader_election.cc:288] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769), ea767071649f45fab000280c6b143939 (127.1.19.61:34117)
01:28:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:00.903324 1485 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:28:00.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:00.907454 1485 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769): Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:28:00.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:00.908272 1484 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea767071649f45fab000280c6b143939 (127.1.19.61:34117): Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:28:00.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:00.908636 1484 leader_election.cc:302] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bfb241e1d7bf41f3b63602831cdccfcd; no voters: b5edb09270444211922ae5dcc10b3be2, ea767071649f45fab000280c6b143939
01:28:00.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:00.909299 1676 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:28:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:01.813319 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:28:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:01.813619 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:28:02.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:28:02.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:28:02.375 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:02.375 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:02.375 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:02.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:02.375 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:02.376 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:02.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:28:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:02.817085 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:28:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:02.817390 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:28:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:03.820726 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:28:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:03.821039 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:28:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:04.340214 1683 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:28:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:04.340922 1683 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b5edb09270444211922ae5dcc10b3be2" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 46769 } } peers { permanent_uuid: "ea767071649f45fab000280c6b143939" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 34117 } } peers { permanent_uuid: "bfb241e1d7bf41f3b63602831cdccfcd" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 33973 } }
01:28:04.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:04.343060 1683 leader_election.cc:288] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769), ea767071649f45fab000280c6b143939 (127.1.19.61:34117)
01:28:04.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:04.350427 1485 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b5edb09270444211922ae5dcc10b3be2 (127.1.19.62:46769): Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111)
01:28:04.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:04.351195 1484 leader_election.cc:334] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea767071649f45fab000280c6b143939 (127.1.19.61:34117): Network error: Client connection negotiation failed: client connection to 127.1.19.61:34117: connect: Connection refused (error 111)
01:28:04.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:04.351656 1484 leader_election.cc:302] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: bfb241e1d7bf41f3b63602831cdccfcd; no voters: b5edb09270444211922ae5dcc10b3be2, ea767071649f45fab000280c6b143939
01:28:04.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:04.352385 1683 raft_consensus.cc:2726] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:28:04.632 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:28:04.633 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:28:04.634 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:04.634 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:04.634 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:04.634 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:04.634 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:04.635 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:04.638 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:28:04.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:04.824950 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:28:04.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:04.825277 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:28:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 01:28:05 dist-test-slave-wg14 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781918885, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:28:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:05.349282 1478 init.cc:295] Successfully reacquired a new kerberos TGT
01:28:05.572 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:28:05.573 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:28:05.574 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:05.574 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:05.576 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:05.574 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:05.576 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:05.576 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:05.579 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:28:05.652 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:28:05.653 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:28:05.655 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:05.655 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:05.655 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:05.655 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:05.655 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:05.656 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:05.658 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:28:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:05.776728 1462 master_runner.cc:186] Error getting master registration for 127.1.19.62:46769: Network error: Client connection negotiation failed: client connection to 127.1.19.62:46769: connect: Connection refused (error 111),
01:28:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:05.777288 1462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:28:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:05.828100 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:28:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:05.828392 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:28:06.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:06.831094 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfb241e1d7bf41f3b63602831cdccfcd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:28:06.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:06.831468 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:28:06.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:34117(127.1.19.61:34117)] connecting to peer
01:28:06.854 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:46769(127.1.19.62:46769)] connecting to peer
01:28:06.855 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:06.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:46769(127.1.19.62:46769)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:06.856 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:46769: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769
01:28:06.856 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:06.856 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:34117(127.1.19.61:34117)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:06.856 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:34117: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
01:28:06.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769; will retry
01:28:06.866 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28931), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [204ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [256ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [265ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [713ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [721ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [734ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [743ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [752ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [782ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [821ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [827ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [833ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [841ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [842ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [843ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [914ms] refreshing cache from master, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [920ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [924ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [974ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [986ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [987ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1174ms] refreshing cache from master, [1174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1191ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1192ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1294ms] refreshing cache from master, [1294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1994ms] refreshing cache from master, [1994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2004ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2134ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [6014ms] refreshing cache from master, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [6016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [6019ms] too many traces: truncated at 100 traces, deferred=Deferred@93667329(state=PENDING, result=null, callback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215), errback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215)))
01:28:06.873 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.60:33973(127.1.19.60:33973)] disconnecting while in state READY
01:28:06.874 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.61:34117(127.1.19.61:34117)] disconnecting while in state TERMINATED
01:28:06.874 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.62:46769(127.1.19.62:46769)] disconnecting while in state TERMINATED
01:28:06.880 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:28:06.883 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.19.60:33973(127.1.19.60:33973)] handling channelInactive
01:28:06.883 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.60:33973(127.1.19.60:33973)] cleaning up while in state READY due to: connection closed
01:28:06.884 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:28:06.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:06.893635 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 1462
01:28:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:07.335652 946 debug-util.cc:398] Leaking SignalData structure 0x7b08000af8c0 after lost signal to thread 819
01:28:07.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 01:28:07.337193 946 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9e80 after lost signal to thread 949
01:28:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:28:08Z chronyd exiting
01:28:09.131 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:28:09.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:09.132614 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 656
01:28:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:09.156924 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 818
01:28:09.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:09.187070 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 959
01:28:09.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:09.213753 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 386
01:28:09.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:09.247367 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 456
01:28:09.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 01:28:09.278973 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9_3e26/build/tsan/bin/kudu with pid 531
01:28:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T01:28:10Z chronyd exiting
01:28:11.434 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28931), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [204ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [256ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [265ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [713ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [721ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [734ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [743ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [752ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [782ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [821ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [827ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [833ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [841ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [842ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [843ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [914ms] refreshing cache from master, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [920ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [924ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [974ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [986ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [987ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1174ms] refreshing cache from master, [1174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1191ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1192ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1294ms] refreshing cache from master, [1294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1994ms] refreshing cache from master, [1994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2004ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2134ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [6014ms] refreshing cache from master, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [6016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [6019ms] too many traces: truncated at 100 traces, deferred=Deferred@93667329(state=PENDING, result=null, callback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215), errback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.61:34117
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28931), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [204ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [256ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [265ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [713ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [721ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [734ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [743ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [752ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [782ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [821ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [827ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [833ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [841ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [842ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [843ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [914ms] refreshing cache from master, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [920ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [924ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [974ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [986ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [987ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1174ms] refreshing cache from master, [1174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1191ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1192ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1294ms] refreshing cache from master, [1294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1994ms] refreshing cache from master, [1994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2004ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2134ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [6014ms] refreshing cache from master, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [6016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [6019ms] too many traces: truncated at 100 traces, deferred=Deferred@93667329(state=PENDING, result=null, callback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215), errback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.61:34117
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
01:28:11.940 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
01:28:11.940 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28931), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [204ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [256ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [265ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [713ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [721ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [734ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [743ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [752ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [782ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [821ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [827ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [833ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [841ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [842ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [843ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [914ms] refreshing cache from master, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [920ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [924ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [974ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [986ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [987ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1174ms] refreshing cache from master, [1174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1191ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1192ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1294ms] refreshing cache from master, [1294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1994ms] refreshing cache from master, [1994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2004ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2134ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [6014ms] refreshing cache from master, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [6016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [6019ms] too many traces: truncated at 100 traces, deferred=Deferred@93667329(state=PENDING, result=null, callback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215), errback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.61:34117
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28931), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [204ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [256ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [265ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [713ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [721ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [734ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [743ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [747ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [752ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [774ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [782ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [800ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [821ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [825ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [827ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [833ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [841ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [842ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [843ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [914ms] refreshing cache from master, [914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [920ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [924ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [974ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [986ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [987ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1174ms] refreshing cache from master, [1174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1188ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1191ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1192ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1294ms] refreshing cache from master, [1294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [1300ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1994ms] refreshing cache from master, [1994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [1995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2004ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2134ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [2136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:46769: Network error: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769, [2140ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:34117: Network error: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:33973: OK, [2144ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117, [6014ms] refreshing cache from master, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:34117, [6015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:33973, [6016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:46769, [6019ms] too many traces: truncated at 100 traces, deferred=Deferred@93667329(state=PENDING, result=null, callback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215), errback=(continuation of Deferred@688006843 after retry RPC after error@1588829466) -> (continuation of Deferred@717186671 after retry RPC after error@84258417) -> (continuation of Deferred@1024096148 after retry RPC after error@2131981922) -> (continuation of Deferred@785316849 after retry RPC after error@101760639) -> (continuation of Deferred@1749606398 after retry RPC after error@210502899) -> (continuation of Deferred@1430550496 after retry RPC after error@1445255281) -> (continuation of Deferred@2017286756 after retry RPC after error@343564812) -> (continuation of Deferred@1660419092 after retry RPC after error@316692331) -> (continuation of Deferred@10273065 after retry RPC after error@1202829620) -> (continuation of Deferred@452717194 after retry RPC after error@179350961) -> (continuation of Deferred@1816371271 after retry RPC after error@644996322) -> (continuation of Deferred@1671159068 after retry RPC after error@1816738299) -> (continuation of Deferred@502922669 after retry RPC after error@893887548) -> (continuation of Deferred@134868149 after retry RPC after error@757762734) -> (continuation of Deferred@2146180659 after retry RPC after error@2005021622) -> (continuation of Deferred@816387511 after retry RPC after error@1227855606) -> (continuation of Deferred@1445794918 after retry RPC after error@810848918) -> (continuation of Deferred@1210592646 after retry RPC after error@2097515091) -> (continuation of Deferred@1127559089 after retry RPC after error@1474418019) -> (continuation of Deferred@1579980292 after retry RPC after error@254466176) -> (continuation of Deferred@1695037043 after retry RPC after error@1016839548) -> (continuation of Deferred@104693853 after retry RPC after error@1602922178) -> (continuation of Deferred@2065017070 after retry RPC after error@436509357) -> (continuation of Deferred@1884413656 after retry RPC after error@2053622438) -> (continuation of Deferred@744706012 after retry RPC after error@302803215)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:34117,127.1.19.60:33973,127.1.19.62:46769) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:46769(127.1.19.62:46769): Connection refused: /127.1.19.62:46769,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:34117(127.1.19.61:34117): Connection refused: /127.1.19.61:34117
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.61:34117
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)