15:18:33.922 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:18:33.934 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskTQhlBc/test-tmp
15:18:33.937 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:18:33.941 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskTQhlBc/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:18:34.151 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:18:34.328 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:18:34.329 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:18:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.334267 374 mini_oidc.cc:152] Starting JWKS server
15:18:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.339387 374 webserver.cc:460] Webserver started at http://127.0.0.1:34559/ using document root <none> and password file <none>
15:18:34.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.340032 374 mini_oidc.cc:189] Starting OIDC Discovery server
15:18:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.342303 374 webserver.cc:460] Webserver started at http://127.0.0.1:36627/ using document root <none> and password file <none>
15:18:34.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:18:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:18:34.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:18:34Z Disabled control of system clock
15:18:34.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.400952 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:34.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:34.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/wal
15:18:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/data
15:18:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/logs
15:18:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/data/info.pb
15:18:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:35371
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34559/jwks.json with env {}
15:18:34.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.706781 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:34.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.707355 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:34.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.708040 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:34.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.739208 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.739630 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.739941 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.740279 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.740626 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.740890 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34559/jwks.json
15:18:34.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.775547 386 master_runner.cc:386] Master server non-default flags:
15:18:34.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:34.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:34.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:34.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:34.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/data
15:18:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/wal
15:18:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:35371
15:18:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34559/jwks.json
15:18:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/data/info.pb
15:18:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/logs
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:34.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:34.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:34.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:34.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:34.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:34.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:34.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:34.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:34.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.777124 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:34.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.779240 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:36.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.798695 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:36.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.798635 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:36.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:34.798642 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:36.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:34.799561 386 server_base.cc:1034] running on GCE node
15:18:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.006422 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:36.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.009672 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:36.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.011132 386 hybrid_clock.cc:648] HybridClock initialized: now 1781968716011100 us; error 52 us; skew 500 ppm
15:18:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.011929 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:36.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.024637 386 webserver.cc:460] Webserver started at http://127.0.93.190:36427/ using document root <none> and password file <none>
15:18:36.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.025579 386 fs_manager.cc:359] Metadata directory not provided
15:18:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.025760 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.026137 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.031212 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/data/instance:
15:18:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3bd58004b024d6286692c1abccf6a24"
15:18:36.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:36 on dist-test-slave-9sg2"
15:18:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.032995 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/wal/instance:
15:18:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3bd58004b024d6286692c1abccf6a24"
15:18:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:36 on dist-test-slave-9sg2"
15:18:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.040635 386 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
15:18:36.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.046764 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.047977 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
15:18:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.048337 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/wal
15:18:36.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3bd58004b024d6286692c1abccf6a24"
15:18:36.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:36 on dist-test-slave-9sg2"
15:18:36.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.048657 386 fs_report.cc:383] FS layout report
15:18:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/wal
15:18:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/wal
15:18:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/data/data
15:18:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:36.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.102437 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:36.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.103952 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:36.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.104417 386 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:36.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.181949 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:35371
15:18:36.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.182047 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:35371 every 8 connection(s)
15:18:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.184574 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/data/info.pb
15:18:36.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.189003 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 386
15:18:36.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.189534 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-0/wal/instance
15:18:36.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.191116 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.194070 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/wal
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/data
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/logs
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/data/info.pb
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:40141
15:18:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
15:18:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:36.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34559/jwks.json with env {}
15:18:36.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.205591 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } has no permanent_uuid. Determining permanent_uuid...
15:18:36.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.239459 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } has no permanent_uuid. Determining permanent_uuid...
15:18:36.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.242113 404 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40141: connect: Connection refused (error 111)
15:18:36.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.246881 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40141: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40141: connect: Connection refused (error 111)
15:18:36.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.296427 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } attempt: 1
15:18:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.301271 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40141: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40141: connect: Connection refused (error 111)
15:18:36.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.369669 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } attempt: 2
15:18:36.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.374325 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40141: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40141: connect: Connection refused (error 111)
15:18:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.465759 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } attempt: 3
15:18:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.470211 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40141: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40141: connect: Connection refused (error 111)
15:18:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.520344 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.520907 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.521456 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.552059 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.552367 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:36.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.552567 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:36.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.552742 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:36.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.552911 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:36.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.553037 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34559/jwks.json
15:18:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.587128 456 master_runner.cc:386] Master server non-default flags:
15:18:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/data
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/wal
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:40141
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34559/jwks.json
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/data/info.pb
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/logs
15:18:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.588390 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.590260 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.611423 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.613734 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } attempt: 4
15:18:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.617622 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40141: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40141: connect: Connection refused (error 111)
15:18:36.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.917099 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } attempt: 5
15:18:36.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.921722 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40141: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40141: connect: Connection refused (error 111)
15:18:37.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.469141 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } attempt: 6
15:18:37.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:37.477342 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40141: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40141: connect: Connection refused (error 111)
15:18:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.612710 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:36.613655 456 server_base.cc:1034] running on GCE node
15:18:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:36.611485 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:37.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.843326 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:37.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.846110 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.847507 456 hybrid_clock.cc:648] HybridClock initialized: now 1781968717847448 us; error 70 us; skew 500 ppm
15:18:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.848711 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.856236 456 webserver.cc:460] Webserver started at http://127.0.93.189:35405/ using document root <none> and password file <none>
15:18:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.857435 456 fs_manager.cc:359] Metadata directory not provided
15:18:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.857717 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.858281 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.863477 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/data/instance:
15:18:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9b6dee57bce46038c57949f41d7b6c3"
15:18:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:37 on dist-test-slave-9sg2"
15:18:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.864756 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/wal/instance:
15:18:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9b6dee57bce46038c57949f41d7b6c3"
15:18:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:37 on dist-test-slave-9sg2"
15:18:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.872921 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
15:18:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.878548 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:37.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.879961 456 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:18:37.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.880344 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/wal
15:18:37.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9b6dee57bce46038c57949f41d7b6c3"
15:18:37.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:37 on dist-test-slave-9sg2"
15:18:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.880741 456 fs_report.cc:383] FS layout report
15:18:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/wal
15:18:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/wal
15:18:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/data/data
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:37.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.924048 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:37.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.925534 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:37.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:37.925945 456 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:38.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.006918 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:40141
15:18:38.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.007009 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:40141 every 8 connection(s)
15:18:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.009804 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/data/info.pb
15:18:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.015730 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 456
15:18:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.016095 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.016216 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-1/wal/instance
15:18:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.019202 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/wal
15:18:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/data
15:18:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/logs
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/data/info.pb
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:33531
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34559/jwks.json with env {}
15:18:38.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.031206 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } has no permanent_uuid. Determining permanent_uuid...
15:18:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.064239 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } has no permanent_uuid. Determining permanent_uuid...
15:18:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.087189 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } has no permanent_uuid. Determining permanent_uuid...
15:18:38.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.090761 480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.097429 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.147038 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 1
15:18:38.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.151449 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.219928 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 2
15:18:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.223771 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.315253 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 3
15:18:38.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.319025 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.362560 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.363121 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.363811 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.395393 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.395673 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.395929 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.396124 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.396369 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.396531 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34559/jwks.json
15:18:38.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.431324 531 master_runner.cc:386] Master server non-default flags:
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/data
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/wal
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:33531
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34559/jwks.json
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/data/info.pb
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:38.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/logs
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.432578 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.434645 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:38.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.456218 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:38.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.462487 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 4
15:18:38.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.466773 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.537822 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } attempt: 7
15:18:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.555555 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } has no permanent_uuid. Determining permanent_uuid...
15:18:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.563714 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.622167 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 1
15:18:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.627301 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.708724 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 2
15:18:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.713532 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.766228 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 5
15:18:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.771620 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.798949 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 3
15:18:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.803718 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.944155 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 4
15:18:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.949672 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:39.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.233089 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 5
15:18:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:39.238337 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:39.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.319299 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 6
15:18:39.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:39.326992 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.456437 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:38.456241 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:38.458499 531 server_base.cc:1034] running on GCE node
15:18:39.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.672649 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:39.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.675174 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:39.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.676545 531 hybrid_clock.cc:648] HybridClock initialized: now 1781968719676516 us; error 40 us; skew 500 ppm
15:18:39.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.677449 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:39.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.684944 531 webserver.cc:460] Webserver started at http://127.0.93.188:43439/ using document root <none> and password file <none>
15:18:39.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.686364 531 fs_manager.cc:359] Metadata directory not provided
15:18:39.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.686663 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:39.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.687045 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.692219 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/data/instance:
15:18:39.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "008d58484e46457ba3dba6ec21f48d68"
15:18:39.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:39 on dist-test-slave-9sg2"
15:18:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.693729 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/wal/instance:
15:18:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "008d58484e46457ba3dba6ec21f48d68"
15:18:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:39 on dist-test-slave-9sg2"
15:18:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.701702 531 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
15:18:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.708382 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:39.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.709650 531 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
15:18:39.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.709941 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/wal
15:18:39.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "008d58484e46457ba3dba6ec21f48d68"
15:18:39.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:39 on dist-test-slave-9sg2"
15:18:39.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.710311 531 fs_report.cc:383] FS layout report
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/wal
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/wal
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/data/data
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:39.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.757833 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.759332 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.759764 531 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:39.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.790778 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 6
15:18:39.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:39.798206 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33531: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33531: connect: Connection refused (error 111)
15:18:39.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.845518 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:33531
15:18:39.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.845597 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:33531 every 8 connection(s)
15:18:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.848301 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/data/info.pb
15:18:39.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.855037 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:39.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.857393 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 531
15:18:39.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.857874 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/master-2/wal/instance
15:18:39.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.874547 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } has no permanent_uuid. Determining permanent_uuid...
15:18:39.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.904508 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } has no permanent_uuid. Determining permanent_uuid...
15:18:39.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.915933 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } has no permanent_uuid. Determining permanent_uuid...
15:18:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.933801 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68: Bootstrap starting.
15:18:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.940521 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68: Neither blocks nor log segments found. Creating new log.
15:18:39.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.942194 609 log.cc:824] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68: Log is configured to *not* fsync() on all Append() calls
15:18:39.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.948035 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68: No bootstrap required, opened a new log
15:18:39.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.969667 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:39.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.970476 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:18:39.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.970767 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 008d58484e46457ba3dba6ec21f48d68, State: Initialized, Role: FOLLOWER
15:18:39.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.971676 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:39.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.974535 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:39.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.975155 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: This master's current role is: FOLLOWER
15:18:39.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.975972 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: configured and running, proceeding with master startup.
15:18:40.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:40.006325 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:18:40.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:40.007382 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:18:40.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:40.016580 561 tablet.cc:2369] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:18:40.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:39.999593 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:35371: OK, instance_id { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" instance_seqno: 1781968716161264 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:18:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.035907 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:18:40.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.228807 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:18:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.229398 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.233731 618 leader_election.cc:288] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3bd58004b024d6286692c1abccf6a24 (127.0.93.190:35371), e9b6dee57bce46038c57949f41d7b6c3 (127.0.93.189:40141)
15:18:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.387576 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 7
15:18:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.413571 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3: Bootstrap starting.
15:18:40.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.421558 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3: Neither blocks nor log segments found. Creating new log.
15:18:40.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.424363 529 log.cc:824] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3: Log is configured to *not* fsync() on all Append() calls
15:18:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.429970 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3: No bootstrap required, opened a new log
15:18:40.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.463940 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:40.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.464865 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:18:40.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.465153 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9b6dee57bce46038c57949f41d7b6c3, State: Initialized, Role: FOLLOWER
15:18:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.466092 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:40.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.469202 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:40.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.470188 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [sys.catalog]: This master's current role is: FOLLOWER
15:18:40.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.471624 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [sys.catalog]: configured and running, proceeding with master startup.
15:18:40.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.471802 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "008d58484e46457ba3dba6ec21f48d68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9b6dee57bce46038c57949f41d7b6c3" is_pre_election: true
15:18:40.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.472765 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 008d58484e46457ba3dba6ec21f48d68 in term 0.
15:18:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.474265 557 leader_election.cc:302] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 008d58484e46457ba3dba6ec21f48d68, e9b6dee57bce46038c57949f41d7b6c3; no voters:
15:18:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.477802 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:18:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.478193 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:18:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.478513 618 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 0 FOLLOWER]: Advancing to term 1
15:18:40.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.488289 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:40.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.499445 618 leader_election.cc:288] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [CANDIDATE]: Term 1 election: Requested vote from peers c3bd58004b024d6286692c1abccf6a24 (127.0.93.190:35371), e9b6dee57bce46038c57949f41d7b6c3 (127.0.93.189:40141)
15:18:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.501392 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "008d58484e46457ba3dba6ec21f48d68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e9b6dee57bce46038c57949f41d7b6c3"
15:18:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.502502 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [term 0 FOLLOWER]: Advancing to term 1
15:18:40.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.516723 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 008d58484e46457ba3dba6ec21f48d68 in term 1.
15:18:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.517860 557 leader_election.cc:302] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 008d58484e46457ba3dba6ec21f48d68, e9b6dee57bce46038c57949f41d7b6c3; no voters:
15:18:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.518713 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 1 FOLLOWER]: Leader election won for term 1
15:18:40.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.536723 618 raft_consensus.cc:686] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [term 1 LEADER]: Becoming Leader. State: Replica: 008d58484e46457ba3dba6ec21f48d68, State: Running, Role: LEADER
15:18:40.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.541332 618 consensus_queue.cc:237] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:40.560920 645 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:18:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:40.562278 645 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:18:40.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.574689 643 sys_catalog.cc:455] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 008d58484e46457ba3dba6ec21f48d68. Latest consensus state: current_term: 1 leader_uuid: "008d58484e46457ba3dba6ec21f48d68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:40.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.577412 643 sys_catalog.cc:458] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: This master's current role is: LEADER
15:18:40.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.528954 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:35371: OK, instance_id { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" instance_seqno: 1781968716161264 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:18:40.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.579116 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:18:40.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.582754 648 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
15:18:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.590592 648 catalog_manager.cc:1484] Initializing Kudu cluster ID...
15:18:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.618638 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [term 1 FOLLOWER]: Refusing update from remote peer 008d58484e46457ba3dba6ec21f48d68: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:18:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.621886 643 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:18:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.658147 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 008d58484e46457ba3dba6ec21f48d68. Latest consensus state: current_term: 1 leader_uuid: "008d58484e46457ba3dba6ec21f48d68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.659483 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [sys.catalog]: This master's current role is: FOLLOWER
15:18:40.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.673297 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "008d58484e46457ba3dba6ec21f48d68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:40.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.674130 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: This master's current role is: LEADER
15:18:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.677816 648 catalog_manager.cc:1347] Generated new cluster ID: 6603ff63a12f4e0dad1109f21945e256
15:18:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.678123 648 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
15:18:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.678866 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "008d58484e46457ba3dba6ec21f48d68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.681851 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3 [sys.catalog]: This master's current role is: FOLLOWER
15:18:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.728621 648 catalog_manager.cc:1370] Generated new certificate authority record
15:18:40.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.731885 648 catalog_manager.cc:1504] Loading token signing keys...
15:18:40.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.756866 648 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68: Generated new TSK 0
15:18:40.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.762290 648 catalog_manager.cc:1514] Initializing in-progress tserver states...
15:18:40.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.787391 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/wal
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/data
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/logs
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/data/info.pb
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:18:40.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.831550 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } attempt: 7
15:18:40.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.897383 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24: Bootstrap starting.
15:18:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.919373 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24: Neither blocks nor log segments found. Creating new log.
15:18:40.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.926985 454 log.cc:824] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24: Log is configured to *not* fsync() on all Append() calls
15:18:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:40.953524 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24: No bootstrap required, opened a new log
15:18:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.057612 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.060807 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:18:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.061945 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c3bd58004b024d6286692c1abccf6a24, State: Initialized, Role: FOLLOWER
15:18:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.064990 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } }
15:18:41.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.087589 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "008d58484e46457ba3dba6ec21f48d68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c3bd58004b024d6286692c1abccf6a24" is_pre_election: true
15:18:41.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.089023 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "008d58484e46457ba3dba6ec21f48d68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c3bd58004b024d6286692c1abccf6a24"
15:18:41.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.092226 429 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 008d58484e46457ba3dba6ec21f48d68 in term 0.
15:18:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.100270 427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [term 0 FOLLOWER]: Advancing to term 1
15:18:41.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.107836 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [term 1 FOLLOWER]: Refusing update from remote peer 008d58484e46457ba3dba6ec21f48d68: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:18:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.113236 618 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:18:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:41.117059 408 tablet.cc:2369] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:18:41.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.109992 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [sys.catalog]: configured and running, proceeding with master startup.
15:18:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.119333 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.123970 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [sys.catalog]: This master's current role is: FOLLOWER
15:18:41.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.169768 661 mvcc.cc:205] Tried to move back new op lower bound from 7298943879630118912 to 7298943879386050560. Current Snapshot: MvccSnapshot[applied={T|T < 7298943879630118912}]
15:18:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.182993 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 008d58484e46457ba3dba6ec21f48d68. Latest consensus state: current_term: 1 leader_uuid: "008d58484e46457ba3dba6ec21f48d68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.182921 658 sys_catalog.cc:455] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "008d58484e46457ba3dba6ec21f48d68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.183784 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [sys.catalog]: This master's current role is: FOLLOWER
15:18:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.183816 658 sys_catalog.cc:458] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24 [sys.catalog]: This master's current role is: FOLLOWER
15:18:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.224835 673 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:18:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:41.316947 654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:41.317533 654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:41.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:41.318209 654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:41.351058 654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:41.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:41.352094 654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
15:18:41.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.390435 654 tablet_server_runner.cc:78] Tablet server non-default flags:
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/data
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/wal
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:41.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/data/info.pb
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/logs
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:18:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.392061 654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.394404 654 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:41.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:41.435595 679 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:41.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.571476 645 catalog_manager.cc:1259] Loaded cluster ID: 6603ff63a12f4e0dad1109f21945e256
15:18:41.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.573132 645 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3: loading cluster ID for follower catalog manager: success
15:18:41.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.608973 645 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3: acquiring CA information for follower catalog manager: success
15:18:41.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.619479 645 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e9b6dee57bce46038c57949f41d7b6c3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:18:41.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.622637 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "008d58484e46457ba3dba6ec21f48d68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bd58004b024d6286692c1abccf6a24" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 35371 } } peers { permanent_uuid: "e9b6dee57bce46038c57949f41d7b6c3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40141 } } peers { permanent_uuid: "008d58484e46457ba3dba6ec21f48d68" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33531 } } }
15:18:41.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:41.623986 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P 008d58484e46457ba3dba6ec21f48d68 [sys.catalog]: This master's current role is: LEADER
15:18:42.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:42.252847 673 catalog_manager.cc:1259] Loaded cluster ID: 6603ff63a12f4e0dad1109f21945e256
15:18:42.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:42.253293 673 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24: loading cluster ID for follower catalog manager: success
15:18:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:42.274307 673 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24: acquiring CA information for follower catalog manager: success
15:18:42.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:42.297003 673 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c3bd58004b024d6286692c1abccf6a24: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:18:43.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:42.812029 678 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 654
15:18:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:43.636878 678 kernel_stack_watchdog.cc:198] Thread 654 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 376ms:
15:18:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:18:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:18:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:18:43.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:18:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:43.637954 654 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.205s user 0.440s sys 0.858s
15:18:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:43.638304 654 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.205s user 0.440s sys 0.858s
15:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:41.437897 680 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:43.641295 683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:43.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.641476 654 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:43.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:43.642366 681 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2202 milliseconds
15:18:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.644306 654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:43.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.647317 654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:43.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.648824 654 hybrid_clock.cc:648] HybridClock initialized: now 1781968723648715 us; error 123 us; skew 500 ppm
15:18:43.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.649847 654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.661437 654 webserver.cc:460] Webserver started at http://127.0.93.129:42407/ using document root <none> and password file <none>
15:18:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.662412 654 fs_manager.cc:359] Metadata directory not provided
15:18:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.662681 654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:43.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.663187 654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:43.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.668001 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/data/instance:
15:18:43.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15b8f94a7d1042889b1763edf720cf90"
15:18:43.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:43 on dist-test-slave-9sg2"
15:18:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.669340 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/wal/instance:
15:18:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15b8f94a7d1042889b1763edf720cf90"
15:18:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:43 on dist-test-slave-9sg2"
15:18:43.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.678088 654 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.000s sys 0.006s
15:18:43.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.684633 693 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:43.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.685978 654 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
15:18:43.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.686424 654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/wal
15:18:43.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15b8f94a7d1042889b1763edf720cf90"
15:18:43.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:43 on dist-test-slave-9sg2"
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.686806 654 fs_report.cc:383] FS layout report
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/wal
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/wal
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/data/data
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:43.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.744246 654 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:43.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.745801 654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:43.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.746315 654 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:43.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.749930 654 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:18:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.754875 654 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:18:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.755108 654 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:18:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.755411 654 ts_tablet_manager.cc:610] Registered 0 tablets
15:18:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.755605 654 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:18:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.979164 654 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:37979
15:18:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.979271 805 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:37979 every 8 connection(s)
15:18:43.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.982604 654 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/data/info.pb
15:18:43.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.991799 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 654
15:18:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:43.993709 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-0/wal/instance
15:18:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.047856 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/wal
15:18:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/data
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/logs
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/data/info.pb
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:44.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
15:18:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
15:18:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
15:18:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:18:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.109372 806 heartbeater.cc:347] Connected to a master server at 127.0.93.188:33531
15:18:44.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.109890 806 heartbeater.cc:464] Registering TS with master...
15:18:44.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.111142 806 heartbeater.cc:511] Master 127.0.93.188:33531 requested a full tablet report, sending...
15:18:44.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.113677 810 heartbeater.cc:347] Connected to a master server at 127.0.93.190:35371
15:18:44.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.114055 810 heartbeater.cc:464] Registering TS with master...
15:18:44.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.114926 810 heartbeater.cc:511] Master 127.0.93.190:35371 requested a full tablet report, sending...
15:18:44.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.114991 572 ts_manager.cc:194] Registered new tserver with Master: 15b8f94a7d1042889b1763edf720cf90 (127.0.93.129:37979)
15:18:44.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.115772 807 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40141
15:18:44.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.116123 807 heartbeater.cc:464] Registering TS with master...
15:18:44.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.117019 807 heartbeater.cc:511] Master 127.0.93.189:40141 requested a full tablet report, sending...
15:18:44.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.120548 494 ts_manager.cc:194] Registered new tserver with Master: 15b8f94a7d1042889b1763edf720cf90 (127.0.93.129:37979)
15:18:44.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.120537 419 ts_manager.cc:194] Registered new tserver with Master: 15b8f94a7d1042889b1763edf720cf90 (127.0.93.129:37979)
15:18:44.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.122430 572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:54051
15:18:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:44.452797 816 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:44.453356 816 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:44.453981 816 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:44.486797 816 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:44.487746 816 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.523901 816 tablet_server_runner.cc:78] Tablet server non-default flags:
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/data
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/wal
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/data/info.pb
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/logs
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.525404 816 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.527535 816 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:44.551321 824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.127146 806 heartbeater.cc:503] Master 127.0.93.188:33531 was elected leader, sending a full tablet report...
15:18:45.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:44.552142 816 server_base.cc:1034] running on GCE node
15:18:45.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:44.553381 826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:45.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:44.552482 823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.764765 816 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.767154 816 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.768594 816 hybrid_clock.cc:648] HybridClock initialized: now 1781968725768555 us; error 39 us; skew 500 ppm
15:18:45.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.769502 816 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:45.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.777488 816 webserver.cc:460] Webserver started at http://127.0.93.130:41347/ using document root <none> and password file <none>
15:18:45.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.778517 816 fs_manager.cc:359] Metadata directory not provided
15:18:45.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.778769 816 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:45.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.779304 816 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.784389 816 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/data/instance:
15:18:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa209a9e4bac412a94cdf91dc1856eec"
15:18:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:45 on dist-test-slave-9sg2"
15:18:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.785620 816 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/wal/instance:
15:18:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa209a9e4bac412a94cdf91dc1856eec"
15:18:45.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:45 on dist-test-slave-9sg2"
15:18:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.793061 816 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
15:18:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.798900 834 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:45.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.799980 816 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
15:18:45.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.800359 816 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/wal
15:18:45.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa209a9e4bac412a94cdf91dc1856eec"
15:18:45.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:45 on dist-test-slave-9sg2"
15:18:45.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.800719 816 fs_report.cc:383] FS layout report
15:18:45.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/wal
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/wal
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/data/data
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:45.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:45.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.840816 816 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.842872 816 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.843413 816 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:45.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.846804 816 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:18:45.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.853029 816 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:18:45.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.853317 816 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:18:45.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.853611 816 ts_tablet_manager.cc:610] Registered 0 tablets
15:18:45.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:45.853807 816 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:18:46.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.018296 816 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:40715
15:18:46.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.018409 946 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:40715 every 8 connection(s)
15:18:46.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.021098 816 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/data/info.pb
15:18:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.026525 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 816
15:18:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.027316 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-1/wal/instance
15:18:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.046981 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/wal
15:18:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/data
15:18:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:46.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/logs
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/data/info.pb
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
15:18:46.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:18:46.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.084141 948 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40141
15:18:46.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.084823 948 heartbeater.cc:464] Registering TS with master...
15:18:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.086841 948 heartbeater.cc:511] Master 127.0.93.189:40141 requested a full tablet report, sending...
15:18:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.087968 947 heartbeater.cc:347] Connected to a master server at 127.0.93.188:33531
15:18:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.088320 947 heartbeater.cc:464] Registering TS with master...
15:18:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.089219 947 heartbeater.cc:511] Master 127.0.93.188:33531 requested a full tablet report, sending...
15:18:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.092871 494 ts_manager.cc:194] Registered new tserver with Master: aa209a9e4bac412a94cdf91dc1856eec (127.0.93.130:40715)
15:18:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.096650 572 ts_manager.cc:194] Registered new tserver with Master: aa209a9e4bac412a94cdf91dc1856eec (127.0.93.130:40715)
15:18:46.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.099888 572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:53617
15:18:46.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.110474 949 heartbeater.cc:347] Connected to a master server at 127.0.93.190:35371
15:18:46.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.110801 949 heartbeater.cc:464] Registering TS with master...
15:18:46.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.111433 949 heartbeater.cc:511] Master 127.0.93.190:35371 requested a full tablet report, sending...
15:18:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.113579 419 ts_manager.cc:194] Registered new tserver with Master: aa209a9e4bac412a94cdf91dc1856eec (127.0.93.130:40715)
15:18:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:46.397647 955 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:46.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:46.398247 955 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:46.398869 955 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:46.430444 955 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:46.431394 955 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.465838 955 tablet_server_runner.cc:78] Tablet server non-default flags:
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:44971
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/data
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/wal
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/data/info.pb
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:35371,127.0.93.189:40141,127.0.93.188:33531
15:18:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/logs
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.467433 955 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:46.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.469362 955 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:46.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:46.491048 963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:47.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.106483 947 heartbeater.cc:503] Master 127.0.93.188:33531 was elected leader, sending a full tablet report...
15:18:47.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:46.491611 966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:47.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:46.494078 955 server_base.cc:1034] running on GCE node
15:18:47.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:46.492082 964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.690367 955 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:47.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.693250 955 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:47.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.694835 955 hybrid_clock.cc:648] HybridClock initialized: now 1781968727694794 us; error 55 us; skew 500 ppm
15:18:47.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.695659 955 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:47.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.702860 955 webserver.cc:460] Webserver started at http://127.0.93.131:43191/ using document root <none> and password file <none>
15:18:47.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.703876 955 fs_manager.cc:359] Metadata directory not provided
15:18:47.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.704073 955 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:47.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.704551 955 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:47.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.710005 955 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/data/instance:
15:18:47.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5faa2df70d0a4798a3f6a91b3c04d81f"
15:18:47.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:47 on dist-test-slave-9sg2"
15:18:47.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.711047 955 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/wal/instance:
15:18:47.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5faa2df70d0a4798a3f6a91b3c04d81f"
15:18:47.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:47 on dist-test-slave-9sg2"
15:18:47.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.718226 955 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
15:18:47.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.724974 974 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:47.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.726042 955 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:18:47.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.726402 955 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/wal
15:18:47.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5faa2df70d0a4798a3f6a91b3c04d81f"
15:18:47.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:47 on dist-test-slave-9sg2"
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.726778 955 fs_report.cc:383] FS layout report
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/wal
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/wal
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/data/data
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:47.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.775795 955 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:47.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.777341 955 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:47.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.777932 955 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:47.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.780649 955 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:18:47.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.784807 955 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:18:47.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.785044 955 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:18:47.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.785393 955 ts_tablet_manager.cc:610] Registered 0 tablets
15:18:47.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.785588 955 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:18:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.945624 955 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:42421
15:18:47.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.945758 1086 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:42421 every 8 connection(s)
15:18:47.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.948325 955 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/data/info.pb
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.956746 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 955
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:47.957279 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster6796056064775761241/ts-2/wal/instance
15:18:48.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.035487 1088 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40141
15:18:48.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.035979 1088 heartbeater.cc:464] Registering TS with master...
15:18:48.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.037185 1088 heartbeater.cc:511] Master 127.0.93.189:40141 requested a full tablet report, sending...
15:18:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.038511 1089 heartbeater.cc:347] Connected to a master server at 127.0.93.190:35371
15:18:48.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.038828 1089 heartbeater.cc:464] Registering TS with master...
15:18:48.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.039580 1089 heartbeater.cc:511] Master 127.0.93.190:35371 requested a full tablet report, sending...
15:18:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.042809 419 ts_manager.cc:194] Registered new tserver with Master: 5faa2df70d0a4798a3f6a91b3c04d81f (127.0.93.131:42421)
15:18:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.042997 494 ts_manager.cc:194] Registered new tserver with Master: 5faa2df70d0a4798a3f6a91b3c04d81f (127.0.93.131:42421)
15:18:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.044584 1087 heartbeater.cc:347] Connected to a master server at 127.0.93.188:33531
15:18:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.044942 1087 heartbeater.cc:464] Registering TS with master...
15:18:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.045819 1087 heartbeater.cc:511] Master 127.0.93.188:33531 requested a full tablet report, sending...
15:18:48.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.048280 571 ts_manager.cc:194] Registered new tserver with Master: 5faa2df70d0a4798a3f6a91b3c04d81f (127.0.93.131:42421)
15:18:48.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.050794 571 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:56655
15:18:48.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.069412 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
15:18:48.080 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:18:48.081 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:18:48.092 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 35371
}
bound_http_address {
host: "127.0.93.190"
port: 36427
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 40141
}
bound_http_address {
host: "127.0.93.189"
port: 35405
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 33531
}
bound_http_address {
host: "127.0.93.188"
port: 43439
}
}
}
15:18:48.097 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:18:48.100 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 37979
}
bound_http_address {
host: "127.0.93.129"
port: 42407
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 40715
}
bound_http_address {
host: "127.0.93.130"
port: 41347
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 42421
}
bound_http_address {
host: "127.0.93.131"
port: 43191
}
}
}
15:18:48.100 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:18:48.319 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:18:48.441 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskTQhlBc/test-tmp
15:18:48.441 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:18:48.441 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskTQhlBc/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:18:48.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:18:48.807 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:48.807 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:18:48.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.810993 1100 mini_oidc.cc:152] Starting JWKS server
15:18:48.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.814486 1100 webserver.cc:460] Webserver started at http://127.0.0.1:45275/ using document root <none> and password file <none>
15:18:48.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.814980 1100 mini_oidc.cc:189] Starting OIDC Discovery server
15:18:48.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:48.816840 1100 webserver.cc:460] Webserver started at http://127.0.0.1:40125/ using document root <none> and password file <none>
15:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/principal' for realm 'KRBTEST.COM',
15:18:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:18:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:48 dist-test-slave-9sg2 krb5kdc[1112](info): setting up network...
15:18:48.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:18:48.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:48 dist-test-slave-9sg2 krb5kdc[1112](info): set up 2 sockets
15:18:48.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:48 dist-test-slave-9sg2 krb5kdc[1112](info): commencing operation
15:18:48.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:18:49.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:49.055065 1087 heartbeater.cc:503] Master 127.0.93.188:33531 was elected leader, sending a full tablet report...
15:18:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:49.966316 1100 mini_kdc.cc:121] Time spent starting KDC: real 1.125s user 0.005s sys 0.017s
15:18:49.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:18:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:18:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:18:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:18:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:18:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:18:50.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:18:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:18:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:18:50.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:18:50.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/test-user.keytab.
15:18:50.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/test-user.keytab.
15:18:50.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:50 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968730, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:18:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:18:50.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:18:50Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:18:50.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:18:50Z Disabled control of system clock
15:18:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.62@KRBTEST.COM; defaulting to no policy
15:18:50.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:18:50.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.62@KRBTEST.COM" created.
15:18:50.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:18:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.62.keytab.
15:18:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.62.keytab.
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:50.179175 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:50.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/logs
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/info.pb
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:45947
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.62.keytab
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:18:50.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:50.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:50.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:50.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:50.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:50.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:50.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:50.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:50.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:18:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.517190 1125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.517745 1125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.518342 1125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:50.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.553301 1125 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:18:50.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.553536 1125 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:50.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.553787 1125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.553977 1125 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.554167 1125 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.554360 1125 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.554498 1125 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45275/jwks.json
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:50.592553 1125 master_runner.cc:386] Master server non-default flags:
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.62.keytab
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:45947
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/info.pb
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/logs
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:50.593673 1125 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:50.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:50.594911 1125 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.614913 1131 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.623497 1083 debug-util.cc:398] Leaking SignalData structure 0x7b08000af140 after lost signal to thread 958
15:18:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.624387 1083 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba000 after lost signal to thread 1086
15:18:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.615096 1132 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:50.615089 1134 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:50.616607 1125 server_base.cc:1034] running on GCE node
15:18:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.914458 1125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:51.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.917992 1125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.919694 1125 hybrid_clock.cc:648] HybridClock initialized: now 1781968731919610 us; error 82 us; skew 500 ppm
15:18:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.920965 1125 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:51 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968731, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:18:51.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.928790 1125 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
15:18:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.935184 1125 webserver.cc:460] Webserver started at http://127.1.19.62:40157/ using document root <none> and password file <none>
15:18:51.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.936460 1125 fs_manager.cc:359] Metadata directory not provided
15:18:51.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.936736 1125 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:51.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.937304 1125 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:51.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.945379 1125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/instance:
15:18:51.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "156c9f3d91914579ba160f8878d46675"
15:18:51.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:51 on dist-test-slave-9sg2"
15:18:51.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.947109 1125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal/instance:
15:18:51.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "156c9f3d91914579ba160f8878d46675"
15:18:51.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:51 on dist-test-slave-9sg2"
15:18:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.957799 1125 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.010s sys 0.000s
15:18:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.965749 1143 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.967170 1125 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.967595 1125 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "156c9f3d91914579ba160f8878d46675"
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:51 on dist-test-slave-9sg2"
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:51.968058 1125 fs_report.cc:383] FS layout report
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/data
15:18:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.017508 1125 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.026026 1125 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.026852 1125 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:52.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.143983 1125 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:45947
15:18:52.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.144059 1194 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:45947 every 8 connection(s)
15:18:52.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.148491 1125 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/info.pb
15:18:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.155712 1195 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:52.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.156376 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 1125
15:18:52.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.157006 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal/instance
15:18:52.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.174448 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } has no permanent_uuid. Determining permanent_uuid...
15:18:52.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.61@KRBTEST.COM; defaulting to no policy
15:18:52.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:18:52.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.61@KRBTEST.COM" created.
15:18:52.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:52 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968731, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
15:18:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:18:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.61.keytab.
15:18:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.61.keytab.
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.227041 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/logs
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/info.pb
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:46569
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.61.keytab
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:18:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.240913 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } has no permanent_uuid. Determining permanent_uuid...
15:18:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.243618 1144 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:18:52.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.248456 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:46569: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:18:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.297993 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } attempt: 1
15:18:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.302711 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:46569: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:18:52.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.371105 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } attempt: 2
15:18:52.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.375348 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:46569: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:18:52.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.466794 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } attempt: 3
15:18:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.472291 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:46569: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:18:52.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.615758 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } attempt: 4
15:18:52.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.619994 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:46569: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:18:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.628083 1201 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.628734 1201 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:52.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.629395 1201 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:52.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.663117 1201 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:18:52.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.663406 1201 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:52.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.663669 1201 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:52.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.663877 1201 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:52.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.664110 1201 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:52.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.664335 1201 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:52.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.664476 1201 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45275/jwks.json
15:18:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.704212 1201 master_runner.cc:386] Master server non-default flags:
15:18:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data
15:18:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.61.keytab
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:46569
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/info.pb
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/logs
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.706136 1201 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:52.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.707535 1201 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:52.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.728579 1211 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:52.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.919445 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } attempt: 5
15:18:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.924404 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:46569: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:18:53.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.471800 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } attempt: 6
15:18:53.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:53.480060 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:46569: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:18:53.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.729738 1208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:53.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:52.728569 1209 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:53.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:52.730592 1201 server_base.cc:1034] running on GCE node
15:18:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.968855 1201 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.972571 1201 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.974009 1201 hybrid_clock.cc:648] HybridClock initialized: now 1781968733973961 us; error 75 us; skew 500 ppm
15:18:53.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.975036 1201 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:53.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:53 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968733, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:18:53.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.985037 1201 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
15:18:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.990525 1201 webserver.cc:460] Webserver started at http://127.1.19.61:41275/ using document root <none> and password file <none>
15:18:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.991536 1201 fs_manager.cc:359] Metadata directory not provided
15:18:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.991776 1201 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.992360 1201 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.997694 1201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/instance:
15:18:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d92ce64bcc6a4b46b5323593c7d6abba"
15:18:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:53 on dist-test-slave-9sg2"
15:18:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:53.998889 1201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal/instance:
15:18:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d92ce64bcc6a4b46b5323593c7d6abba"
15:18:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:53 on dist-test-slave-9sg2"
15:18:54.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.006723 1201 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
15:18:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.014217 1221 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.015940 1201 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
15:18:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.016374 1201 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:18:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d92ce64bcc6a4b46b5323593c7d6abba"
15:18:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:53 on dist-test-slave-9sg2"
15:18:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.016878 1201 fs_report.cc:383] FS layout report
15:18:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:18:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:18:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/data
15:18:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.060009 1201 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.065430 1201 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:54.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.065830 1201 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:54.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.153642 1201 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:46569
15:18:54.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.153780 1272 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:46569 every 8 connection(s)
15:18:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.156454 1201 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/info.pb
15:18:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.156883 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 1201
15:18:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.157205 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal/instance
15:18:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.164043 1273 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:54.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.60@KRBTEST.COM; defaulting to no policy
15:18:54.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:18:54.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.60@KRBTEST.COM" created.
15:18:54.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.180079 1273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } has no permanent_uuid. Determining permanent_uuid...
15:18:54.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:54 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968733, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
15:18:54.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:18:54.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.60.keytab.
15:18:54.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.60.keytab.
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.212827 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/logs
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/info.pb
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:44387
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.60.keytab
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:54.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:18:54.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.222875 1273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } has no permanent_uuid. Determining permanent_uuid...
15:18:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:54 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968733, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
15:18:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.254287 1273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } has no permanent_uuid. Determining permanent_uuid...
15:18:54.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.256520 1223 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.261122 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.310566 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 1
15:18:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.314538 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.382999 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 2
15:18:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.387339 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.478693 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 3
15:18:54.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.482667 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.540508 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } attempt: 7
15:18:54.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.543296 1279 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:54.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.543942 1279 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:54.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.544698 1279 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:54.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:54 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968731, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
15:18:54.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.563007 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } has no permanent_uuid. Determining permanent_uuid...
15:18:54.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.568233 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.582860 1279 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:18:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.583081 1279 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.583331 1279 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.583523 1279 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.583703 1279 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.583900 1279 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.584038 1279 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45275/jwks.json
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.620677 1279 master_runner.cc:386] Master server non-default flags:
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.60.keytab
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:44387
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/info.pb
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/logs
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.621944 1279 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:54.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.622926 1279 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:54.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.626044 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 4
15:18:54.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.626662 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 1
15:18:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.630388 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.631651 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.641294 1294 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:54.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.713054 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 2
15:18:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.717317 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.802716 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 3
15:18:54.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.806973 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.929811 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 5
15:18:54.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.933275 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:54.947359 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 4
15:18:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.950804 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:55.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.234431 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 5
15:18:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:55.238318 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.480866 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 6
15:18:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:55.487914 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:55.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.790889 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } attempt: 6
15:18:55.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:55.798177 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:44387: Network error: Client connection negotiation failed: client connection to 127.1.19.60:44387: connect: Connection refused (error 111)
15:18:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.641520 1292 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:55.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:54.641398 1291 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:55.894688 1293 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:18:55.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.894942 1279 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:55.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.898988 1279 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.902190 1279 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.903620 1279 hybrid_clock.cc:648] HybridClock initialized: now 1781968735903583 us; error 60 us; skew 500 ppm
15:18:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.904474 1279 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:55.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:55 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968735, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:18:55.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.911489 1279 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
15:18:55.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.917511 1279 webserver.cc:460] Webserver started at http://127.1.19.60:37369/ using document root <none> and password file <none>
15:18:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.918481 1279 fs_manager.cc:359] Metadata directory not provided
15:18:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.918751 1279 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.919260 1279 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:18:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.924214 1279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/instance:
15:18:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad85e9de93cb4343b57e9496676c5f9d"
15:18:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:55 on dist-test-slave-9sg2"
15:18:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.925427 1279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal/instance:
15:18:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad85e9de93cb4343b57e9496676c5f9d"
15:18:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:55 on dist-test-slave-9sg2"
15:18:55.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.933146 1279 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
15:18:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.938539 1307 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.939682 1279 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:18:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.940052 1279 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:18:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad85e9de93cb4343b57e9496676c5f9d"
15:18:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:55 on dist-test-slave-9sg2"
15:18:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.940470 1279 fs_report.cc:383] FS layout report
15:18:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:18:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:18:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/data
15:18:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.979873 1279 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.986084 1279 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:55.986590 1279 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.080837 1279 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:44387
15:18:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.080911 1358 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:44387 every 8 connection(s)
15:18:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.083482 1279 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/info.pb
15:18:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.085233 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 1279
15:18:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.085691 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal/instance
15:18:56.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.091646 1359 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.109120 1359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } has no permanent_uuid. Determining permanent_uuid...
15:18:56.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:56 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968730, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
15:18:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:56 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968735, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
15:18:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.146169 1359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } has no permanent_uuid. Determining permanent_uuid...
15:18:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:56 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968730, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
15:18:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:56 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968735, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
15:18:56.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.183063 1359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } has no permanent_uuid. Determining permanent_uuid...
15:18:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:56 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968730, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
15:18:56.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:56 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968735, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
15:18:56.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.212641 1100 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
15:18:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.218048 1359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: Bootstrap starting.
15:18:56.219 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:56.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:18:56.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.224898 1359 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: Neither blocks nor log segments found. Creating new log.
15:18:56.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.227540 1359 log.cc:824] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: Log is configured to *not* fsync() on all Append() calls
15:18:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.233203 1359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: No bootstrap required, opened a new log
15:18:56.234 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:18:56.235 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:18:56.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.62"
port: 45947
}
bound_http_address {
host: "127.1.19.62"
port: 40157
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.61"
port: 46569
}
bound_http_address {
host: "127.1.19.61"
port: 41275
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.60"
port: 44387
}
bound_http_address {
host: "127.1.19.60"
port: 37369
}
}
}
15:18:56.240 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:18:56.242 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:56.242 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:18:56.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:18:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.264163 1359 raft_consensus.cc:348] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:18:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.264838 1359 raft_consensus.cc:374] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:18:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.265024 1359 raft_consensus.cc:729] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad85e9de93cb4343b57e9496676c5f9d, State: Initialized, Role: FOLLOWER
15:18:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.265727 1359 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:18:56.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.268558 1370 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } } }
15:18:56.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.269685 1370 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [sys.catalog]: This master's current role is: FOLLOWER
15:18:56.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.270174 1359 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [sys.catalog]: configured and running, proceeding with master startup.
15:18:56.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:56 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968736, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:18:56.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:18:56.278 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:56.281 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc
15:18:56.288 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:18:56.289 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc
15:18:56.292 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:18:56.296 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:18:56.298 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:18:56.298 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:18:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.321693 1381 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:18:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.322570 1381 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:18:56.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:56 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968736, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:18:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.296355 1279 master_runner.cc:186] Error getting master registration for 127.1.19.62:45947: OK, instance_id { permanent_uuid: "156c9f3d91914579ba160f8878d46675" instance_seqno: 1781968732106153 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:18:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:18:56.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.334156 1279 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:18:56.339 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:56.339 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc
15:18:56.340 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:18:56.341 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.62:45947
15:18:56.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:18:56.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.343605 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 1125
15:18:56.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:56.365 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.61:46569
15:18:56.366 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:18:56.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.367002 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 1201
15:18:56.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:56.397 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.60:44387
15:18:56.397 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:18:56.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.397756 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 1279
15:18:56.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:56.422 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.62:45947
15:18:56.423 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.425056 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/logs
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/info.pb
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:45947
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40157
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.62.keytab
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:18:56.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.749796 1386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:56.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.750478 1386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:56.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.751220 1386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.785344 1386 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:18:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.785632 1386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.786046 1386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.786386 1386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.786706 1386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:56.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.787014 1386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:56.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.787235 1386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45275/jwks.json
15:18:56.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.825241 1386 master_runner.cc:386] Master server non-default flags:
15:18:56.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:56.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:56.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data
15:18:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:18:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.62.keytab
15:18:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:45947
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/info.pb
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40157
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/logs
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.826671 1386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:56.828002 1386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.845448 1394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:58.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.852133 1393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:58.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:56.846359 1396 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:58.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.155735 1395 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1307 milliseconds
15:18:58.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.155850 1386 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.157271 1386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:58.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.159933 1386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:58.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.161294 1386 hybrid_clock.cc:648] HybridClock initialized: now 1781968738161240 us; error 60 us; skew 500 ppm
15:18:58.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.162446 1386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:58.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:18:58 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968738, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:18:58.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.168388 1386 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
15:18:58.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.174261 1386 webserver.cc:460] Webserver started at http://127.1.19.62:40157/ using document root <none> and password file <none>
15:18:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.177588 1386 fs_manager.cc:359] Metadata directory not provided
15:18:58.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.177788 1386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.190689 1386 fs_manager.cc:709] Time spent opening directory manager: real 0.009s user 0.007s sys 0.001s
15:18:58.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.199437 1404 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:58.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.202360 1386 fs_manager.cc:725] Time spent opening block manager: real 0.009s user 0.004s sys 0.000s
15:18:58.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.202793 1386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "156c9f3d91914579ba160f8878d46675"
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:51 on dist-test-slave-9sg2"
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.205888 1386 fs_report.cc:383] FS layout report
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/data
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.251122 1386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:18:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.256855 1386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.257277 1386 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:58.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.336834 1386 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:45947
15:18:58.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.336894 1455 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:45947 every 8 connection(s)
15:18:58.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.339771 1386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/data/info.pb
15:18:58.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.344609 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 1386
15:18:58.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:18:58.345 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.61:46569
15:18:58.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:18:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.347381 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:18:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:18:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data
15:18:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:58.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/logs
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/info.pb
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:46569
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41275
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.61.keytab
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
15:18:58.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:18:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:18:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:18:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.353641 1456 sys_catalog.cc:263] Verifying existing consensus state
15:18:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 15:18:58.354581 1456 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:18:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.354959 1386 master.cc:537] Master@127.1.19.62:45947 shutting down...
15:18:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.418437 1386 master.cc:552] Master@127.1.19.62:45947 shutdown complete.
15:18:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:18:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.676985 1458 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.677567 1458 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.678231 1458 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.709533 1458 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:18:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.709795 1458 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.710121 1458 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.710387 1458 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.710579 1458 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.710764 1458 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:58.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.710904 1458 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45275/jwks.json
15:18:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.747321 1458 master_runner.cc:386] Master server non-default flags:
15:18:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:18:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:58.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.61.keytab
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:46569
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/info.pb
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41275
15:18:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/logs
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.748785 1458 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:58.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:58.750169 1458 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:18:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.768852 1469 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:59.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.768859 1466 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:59.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:58.769083 1467 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:59.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:18:59.995231 1468 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:18:59.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:18:59.995328 1458 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.000028 1458 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:00.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.002548 1458 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:00.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.003931 1458 hybrid_clock.cc:648] HybridClock initialized: now 1781968740003897 us; error 42 us; skew 500 ppm
15:19:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.004802 1458 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:19:00 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968740, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:00.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.015667 1458 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
15:19:00.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.022540 1458 webserver.cc:460] Webserver started at http://127.1.19.61:41275/ using document root <none> and password file <none>
15:19:00.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.023769 1458 fs_manager.cc:359] Metadata directory not provided
15:19:00.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.024060 1458 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:00.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.034996 1458 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
15:19:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.040591 1477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:00.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.041906 1458 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
15:19:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.042291 1458 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:19:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d92ce64bcc6a4b46b5323593c7d6abba"
15:19:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:53 on dist-test-slave-9sg2"
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.045615 1458 fs_report.cc:383] FS layout report
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/data
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:00.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.094681 1458 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:19:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.101887 1458 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.102453 1458 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:00.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.177654 1458 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:46569
15:19:00.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.177723 1528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:46569 every 8 connection(s)
15:19:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.180487 1458 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/data/info.pb
15:19:00.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.183346 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 1458
15:19:00.184 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:00.185 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.60:44387
15:19:00.185 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.187180 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:00.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/logs
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/info.pb
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:44387
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37369
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.60.keytab
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:19:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:00.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:19:00.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:00.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.192646 1529 sys_catalog.cc:263] Verifying existing consensus state
15:19:00.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 15:19:00.193627 1529 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:19:00.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.194064 1458 master.cc:537] Master@127.1.19.61:46569 shutting down...
15:19:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.259287 1458 master.cc:552] Master@127.1.19.61:46569 shutdown complete.
15:19:00.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:19:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.506449 1531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.507066 1531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.507649 1531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.538882 1531 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:19:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.539114 1531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:19:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.539390 1531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.539593 1531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:19:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.539827 1531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:19:00.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.540122 1531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:19:00.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.540385 1531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45275/jwks.json
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.579177 1531 master_runner.cc:386] Master server non-default flags:
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:45369
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/krb5kdc/kudu_127.1.19.60.keytab
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:44387
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45275/jwks.json
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/info.pb
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37369
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/logs
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.580363 1531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:00.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.581323 1531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:19:00.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.598493 1540 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.598512 1537 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:00.603518 1531 server_base.cc:1034] running on GCE node
15:19:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:00.598522 1538 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:01.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.830571 1531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:01.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.833082 1531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:01.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.834447 1531 hybrid_clock.cc:648] HybridClock initialized: now 1781968741834417 us; error 42 us; skew 500 ppm
15:19:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.835270 1531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:19:01 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968741, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.841346 1531 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
15:19:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.847150 1531 webserver.cc:460] Webserver started at http://127.1.19.60:37369/ using document root <none> and password file <none>
15:19:01.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.848184 1531 fs_manager.cc:359] Metadata directory not provided
15:19:01.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.848505 1531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.857017 1531 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
15:19:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.862620 1549 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.864215 1531 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
15:19:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.864694 1531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data,/tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:19:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad85e9de93cb4343b57e9496676c5f9d"
15:19:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:18:55 on dist-test-slave-9sg2"
15:19:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.868034 1531 fs_report.cc:383] FS layout report
15:19:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/wal
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/data
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.908315 1531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:19:01.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.914429 1531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:01.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.914839 1531 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.989310 1531 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:44387
15:19:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.989423 1600 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:44387 every 8 connection(s)
15:19:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:01.993079 1531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskTQhlBc/test-tmp/mini-kudu-cluster7503602950746899831/master-2/data/info.pb
15:19:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.003114 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu as pid 1531
15:19:02.003 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.004972 1601 sys_catalog.cc:263] Verifying existing consensus state
15:19:02.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.011590 1601 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: Bootstrap starting.
15:19:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.056247 1601 log.cc:824] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: Log is configured to *not* fsync() on all Append() calls
15:19:02.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.062390 1601 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:19:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.063149 1601 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: Bootstrap complete.
15:19:02.070 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:02.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.089795 1601 raft_consensus.cc:348] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.092368 1601 raft_consensus.cc:374] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:19:02.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.092659 1601 raft_consensus.cc:729] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad85e9de93cb4343b57e9496676c5f9d, State: Initialized, Role: FOLLOWER
15:19:02.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.093623 1601 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.096423 1605 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } } }
15:19:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.097373 1605 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [sys.catalog]: This master's current role is: FOLLOWER
15:19:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.099169 1601 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [sys.catalog]: configured and running, proceeding with master startup.
15:19:02.134 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:02.144 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.60:44387(127.1.19.60:44387)] connecting to peer
15:19:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:02.150175 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:02.150686 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:02.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:02.169994 1555 tablet.cc:2369] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:19:02.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.150192 1531 master_runner.cc:186] Error getting master registration for 127.1.19.62:45947: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111),
15:19:02.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.172935 1531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.224573 1605 raft_consensus.cc:482] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.225137 1605 raft_consensus.cc:504] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:02.229756 1553 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:02.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.233 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.233 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.234268 1605 leader_election.cc:288] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947), d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569)
15:19:02.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.240 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.19.60:44387(127.1.19.60:44387)] Successfully connected to peer
15:19:02.242 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:02.248310 1550 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569): Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:19:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:02.259614 1553 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947): Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.260356 1553 leader_election.cc:302] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad85e9de93cb4343b57e9496676c5f9d; no voters: 156c9f3d91914579ba160f8878d46675, d92ce64bcc6a4b46b5323593c7d6abba
15:19:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:02.261394 1605 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:19:02.341 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:19:02.351 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.60
15:19:02.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:19:02 dist-test-slave-9sg2 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968736, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
15:19:02.614 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x163948e2, L:/127.0.0.1:34604 - R:/127.1.19.60:44387] using SASL/GSSAPI
15:19:02.634 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:02.649 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:02.651 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:02.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.655 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.655 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.656 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:02.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:02.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:02.671 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.671 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.671 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.675 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.675 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.675 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.677 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:02.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:02.690 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:02.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.697 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.700 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:02.729 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:02.730 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:02.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.735 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.735 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.736 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.738 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:02.748 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:02.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:02.751 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.751 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.754 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.756 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:02.788 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:02.790 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:02.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.792 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.793 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.794 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.799 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:02.889 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:02.890 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:02.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:02.894 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.894 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.895 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:02.897 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:03.028 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:03.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.042 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:03.046 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.046 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.047 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.049 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:03.153710 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:03.154052 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:03.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:03.532 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:03.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.534 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.537 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.537 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.540 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:03.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:03.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:03.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.694 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.695 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.695 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.695 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:03.748 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:03.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:03.750 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.750 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.751 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:03.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:03.755 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:04.095440 1624 raft_consensus.cc:482] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:04.095796 1624 raft_consensus.cc:504] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:04.097252 1624 leader_election.cc:288] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947), d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569)
15:19:04.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:04.104094 1553 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947): Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:04.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:04.104597 1550 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569): Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:19:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:04.104974 1550 leader_election.cc:302] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad85e9de93cb4343b57e9496676c5f9d; no voters: 156c9f3d91914579ba160f8878d46675, d92ce64bcc6a4b46b5323593c7d6abba
15:19:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:04.105577 1624 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:19:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:04.157014 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:04.157297 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:05.160094 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:05.160442 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:06.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:06.163209 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:06.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:06.163467 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:06.165120 1632 raft_consensus.cc:482] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:06.165454 1632 raft_consensus.cc:504] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:06.167106 1632 leader_election.cc:288] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947), d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569)
15:19:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:06.172704 1553 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947): Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:06.173163 1550 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569): Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:19:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:06.173543 1550 leader_election.cc:302] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad85e9de93cb4343b57e9496676c5f9d; no voters: 156c9f3d91914579ba160f8878d46675, d92ce64bcc6a4b46b5323593c7d6abba
15:19:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:06.174103 1632 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:19:07.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:07.166332 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:07.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:07.166592 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:07.175551 1531 master_runner.cc:186] Error getting master registration for 127.1.19.62:45947: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111),
15:19:07.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:07.175940 1531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:07.848 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:07.849 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:07.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:07.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:07.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:07.854 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:07.854 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:07.854 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:07.856 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:08.169416 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:08.169693 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:09.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:09.172739 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:09.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:09.173053 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:09.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:09.804432 1641 raft_consensus.cc:482] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:09.804913 1641 raft_consensus.cc:504] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:09.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:09.806600 1641 leader_election.cc:288] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947), d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569)
15:19:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:09.809432 1553 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111) [suppressed 5 similar messages]
15:19:09.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:09.812345 1550 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569): Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:09.812713 1553 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947): Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:09.813010 1553 leader_election.cc:302] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad85e9de93cb4343b57e9496676c5f9d; no voters: 156c9f3d91914579ba160f8878d46675, d92ce64bcc6a4b46b5323593c7d6abba
15:19:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:09.813519 1641 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:19:10.028 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:10.030 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:10.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:10.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:10.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:10.032 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:10.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:10.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:10.036 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:10.175910 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:10.176167 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:11.179677 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:11.180013 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:11.740239 1647 raft_consensus.cc:482] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:11.740833 1647 raft_consensus.cc:504] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:11.742774 1647 leader_election.cc:288] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947), d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569)
15:19:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:11.750311 1553 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947): Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:11.750991 1550 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569): Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:19:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:11.751410 1550 leader_election.cc:302] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad85e9de93cb4343b57e9496676c5f9d; no voters: 156c9f3d91914579ba160f8878d46675, d92ce64bcc6a4b46b5323593c7d6abba
15:19:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:11.752287 1647 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:19:11.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:19:11 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968751, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:11.845057 1545 init.cc:295] Successfully reacquired a new kerberos TGT
15:19:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:12.178205 1531 master_runner.cc:186] Error getting master registration for 127.1.19.62:45947: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111),
15:19:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:12.178620 1531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:12.182822 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:12.183077 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:12.296886 943 debug-util.cc:398] Leaking SignalData structure 0x7b08000af0a0 after lost signal to thread 818
15:19:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:12.297871 943 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdd80 after lost signal to thread 946
15:19:12.408 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:12.410 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:12.410 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.410 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.411 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.414 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.416 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:12.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:12.530 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:12.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.534 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.534 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.534 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.536 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:12.908 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:12.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:12.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:12.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.914 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:12.915 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:13.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:13.186213 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:13.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:13.187304 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:14.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:14.191088 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:14.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:14.191370 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:14.608 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:14.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:14.610 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:14.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:14.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:14.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:14.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:14.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:14.614 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:14.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:14.970685 1654 raft_consensus.cc:482] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:14.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:14.971094 1654 raft_consensus.cc:504] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:14.972990 1654 leader_election.cc:288] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947), d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569)
15:19:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:14.974907 1553 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111) [suppressed 3 similar messages]
15:19:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:14.978067 1553 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947): Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:14.978448 1550 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569): Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:19:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:14.978827 1550 leader_election.cc:302] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad85e9de93cb4343b57e9496676c5f9d; no voters: 156c9f3d91914579ba160f8878d46675, d92ce64bcc6a4b46b5323593c7d6abba
15:19:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:14.979545 1654 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:19:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:15.194631 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:15.194958 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:16.198060 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:16.198364 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:17.181030 1531 master_runner.cc:186] Error getting master registration for 127.1.19.62:45947: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111),
15:19:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:17.181411 1531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:17.201586 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:17.201951 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:17.728 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:17.730 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:17.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:17.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:17.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:17.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:17.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:17.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:17.735 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:18.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:18.205469 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:18.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:18.205802 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:19.209506 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:19.210151 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:20.213541 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:20.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:20.213797 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:21.217844 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:21.218305 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:21.748 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:21.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:21.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:21.750 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:21.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:21.751 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:21.751 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:21.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:21.755 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:21.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:21.760702 1666 raft_consensus.cc:482] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:21.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:21.761226 1666 raft_consensus.cc:504] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:21.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:21.763132 1666 leader_election.cc:288] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947), d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569)
15:19:21.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:21.765441 1550 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:19:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:21.768451 1553 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947): Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:21.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:21.768872 1550 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569): Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:19:21.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:21.769248 1550 leader_election.cc:302] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad85e9de93cb4343b57e9496676c5f9d; no voters: 156c9f3d91914579ba160f8878d46675, d92ce64bcc6a4b46b5323593c7d6abba
15:19:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:21.770069 1666 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:19:21.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:19:21 dist-test-slave-9sg2 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781968761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:21.847919 1545 init.cc:295] Successfully reacquired a new kerberos TGT
15:19:22.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:22.183455 1531 master_runner.cc:186] Error getting master registration for 127.1.19.62:45947: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111),
15:19:22.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:22.183825 1531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:22.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:22.221662 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:22.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:22.221983 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:23.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:23.224758 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:23.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:23.225047 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:24.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:24.227716 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:24.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:24.228016 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:24.368 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:24.369 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:24.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:24.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:24.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:24.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:24.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:24.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:24.374 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:25.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:25.230878 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:25.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:25.231164 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:26.233882 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:26.234181 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:27.186594 1531 master_runner.cc:186] Error getting master registration for 127.1.19.62:45947: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111),
15:19:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:27.187026 1531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:27.236976 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:27.237287 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:27.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:27.397505 1083 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6f40 after lost signal to thread 958
15:19:27.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:27.398695 1083 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba0a0 after lost signal to thread 1086
15:19:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:27.801954 1676 raft_consensus.cc:482] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:27.802330 1676 raft_consensus.cc:504] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "156c9f3d91914579ba160f8878d46675" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 45947 } } peers { permanent_uuid: "d92ce64bcc6a4b46b5323593c7d6abba" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 46569 } } peers { permanent_uuid: "ad85e9de93cb4343b57e9496676c5f9d" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 44387 } }
15:19:27.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:27.803941 1676 leader_election.cc:288] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947), d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569)
15:19:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:27.806245 1553 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:19:27.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:27.809104 1550 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d92ce64bcc6a4b46b5323593c7d6abba (127.1.19.61:46569): Network error: Client connection negotiation failed: client connection to 127.1.19.61:46569: connect: Connection refused (error 111)
15:19:27.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:27.809603 1553 leader_election.cc:334] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 156c9f3d91914579ba160f8878d46675 (127.1.19.62:45947): Network error: Client connection negotiation failed: client connection to 127.1.19.62:45947: connect: Connection refused (error 111)
15:19:27.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:27.809973 1553 leader_election.cc:302] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad85e9de93cb4343b57e9496676c5f9d; no voters: 156c9f3d91914579ba160f8878d46675, d92ce64bcc6a4b46b5323593c7d6abba
15:19:27.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:27.810657 1676 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:19:28.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:28.239944 1616 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad85e9de93cb4343b57e9496676c5f9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:28.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:19:28.240219 1616 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:28.288 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:28.289 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:28.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:28.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:28.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:28.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:28.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:28.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:28.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:28.568 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:45947(127.1.19.62:45947)] connecting to peer
15:19:28.569 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:46569(127.1.19.61:46569)] connecting to peer
15:19:28.570 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:28.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:45947(127.1.19.62:45947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:28.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:45947: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947
15:19:28.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:28.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:46569(127.1.19.61:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:28.571 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:46569: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
15:19:28.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387; will retry
15:19:28.577 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26566), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [225ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [230ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [651ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [667ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [686ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [689ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [725ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [783ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [787ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1029ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1031ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1033ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1035ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1037ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1038ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1516ms] refreshing cache from master, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1529ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1530ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1676ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [5836ms] refreshing cache from master, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [5839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [5841ms] too many traces: truncated at 100 traces, deferred=Deferred@468391698(state=PENDING, result=null, callback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846), errback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846)))
15:19:28.583 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.61:46569(127.1.19.61:46569)] disconnecting while in state TERMINATED
15:19:28.584 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.62:45947(127.1.19.62:45947)] disconnecting while in state TERMINATED
15:19:28.584 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.60:44387(127.1.19.60:44387)] disconnecting while in state READY
15:19:28.589 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:19:28.591 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.19.60:44387(127.1.19.60:44387)] handling channelInactive
15:19:28.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.60:44387(127.1.19.60:44387)] cleaning up while in state READY due to: connection closed
15:19:28.592 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:19:28.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:28.595414 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 1531
15:19:29.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:19:29Z chronyd exiting
15:19:30.822 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:19:30.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:30.823889 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 654
15:19:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:30.851259 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 816
15:19:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:30.885026 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 955
15:19:30.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:30.910442 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 386
15:19:30.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:30.940378 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 456
15:19:30.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:19:30.971015 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskTQhlBc/build/tsan/bin/kudu with pid 531
15:19:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:19:32Z chronyd exiting
15:19:33.157 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26566), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [225ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [230ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [651ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [667ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [686ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [689ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [725ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [783ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [787ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1029ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1031ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1033ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1035ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1037ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1038ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1516ms] refreshing cache from master, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1529ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1530ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1676ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [5836ms] refreshing cache from master, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [5839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [5841ms] too many traces: truncated at 100 traces, deferred=Deferred@468391698(state=PENDING, result=null, callback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846), errback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.61:46569
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26566), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [225ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [230ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [651ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [667ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [686ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [689ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [725ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [783ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [787ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1029ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1031ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1033ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1035ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1037ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1038ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1516ms] refreshing cache from master, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1529ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1530ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1676ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [5836ms] refreshing cache from master, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [5839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [5841ms] too many traces: truncated at 100 traces, deferred=Deferred@468391698(state=PENDING, result=null, callback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846), errback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.61:46569
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
15:19:33.527 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
15:19:33.527 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26566), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [225ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [230ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [651ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [667ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [686ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [689ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [725ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [783ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [787ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1029ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1031ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1033ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1035ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1037ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1038ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1516ms] refreshing cache from master, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1529ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1530ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1676ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [5836ms] refreshing cache from master, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [5839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [5841ms] too many traces: truncated at 100 traces, deferred=Deferred@468391698(state=PENDING, result=null, callback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846), errback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.61:46569
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26566), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [225ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [230ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [623ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [651ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [667ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [686ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [689ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [725ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [783ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [787ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1029ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1031ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1033ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1035ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1037ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1038ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1516ms] refreshing cache from master, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1529ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1530ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1676ms] refreshing cache from master, [1677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1684ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:45947: Network error: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:46569: Network error: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:44387: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569, [5836ms] refreshing cache from master, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:45947, [5837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:46569, [5839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:44387, [5841ms] too many traces: truncated at 100 traces, deferred=Deferred@468391698(state=PENDING, result=null, callback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846), errback=(continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.62:45947,127.1.19.61:46569,127.1.19.60:44387) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:45947(127.1.19.62:45947): Connection refused: /127.1.19.62:45947,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:46569(127.1.19.61:46569): Connection refused: /127.1.19.61:46569
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.61:46569
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)