08:55:58.002 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:55:58.014 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5KApzg/test-tmp
08:55:58.017 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
08:55:58.020 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5KApzg/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
08:55:58.265 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:55:58.278 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:55:58.279 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:55:58.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.280570 367 mini_oidc.cc:152] Starting JWKS server
08:55:58.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.283051 367 webserver.cc:460] Webserver started at http://127.0.0.1:43845/ using document root <none> and password file <none>
08:55:58.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.283233 367 mini_oidc.cc:189] Starting OIDC Discovery server
08:55:58.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.283572 367 webserver.cc:460] Webserver started at http://127.0.0.1:36889/ using document root <none> and password file <none>
08:55:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T08:55:58Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:55:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T08:55:58Z Disabled control of system clock
08:55:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.300869 367 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/wal
08:55:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/data
08:55:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:55:58.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:55:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:55:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/logs
08:55:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/data/info.pb
08:55:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:55:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:55:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:55:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:55:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.254:45791
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.254
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:55:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:55:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43845/jwks.json with env {}
08:55:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.413575 378 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:55:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.413833 378 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:55:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.413870 378 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:55:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.418396 378 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:55:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.418453 378 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:55:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.418471 378 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:55:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.418488 378 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:55:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.418503 378 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:55:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.418516 378 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43845/jwks.json
08:55:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.423719 378 master_runner.cc:386] Master server non-default flags:
08:55:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/data
08:55:58.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/wal
08:55:58.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.254:45791
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:55:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43845/jwks.json
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/data/info.pb
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.254
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.378
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:55:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/logs
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:55:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:55:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:55:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:55:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.425006 378 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:55:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.426817 378 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:55:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.436853 378 server_base.cc:1034] running on GCE node
08:55:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.436792 383 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.436811 386 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.436789 384 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.437544 378 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:55:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.438674 378 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:55:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.439853 378 hybrid_clock.cc:648] HybridClock initialized: now 1781945758439800 us; error 75 us; skew 500 ppm
08:55:58.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.446421 378 webserver.cc:460] Webserver started at http://127.0.91.254:33661/ using document root <none> and password file <none>
08:55:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.446973 378 fs_manager.cc:359] Metadata directory not provided
08:55:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.447033 378 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:55:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.447208 378 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:55:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.449090 378 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/data/instance:
08:55:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4881984b45041578481cb5edd541251"
08:55:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.449601 378 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/wal/instance:
08:55:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4881984b45041578481cb5edd541251"
08:55:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.453375 378 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
08:55:58.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.455830 392 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:55:58.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.457156 378 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
08:55:58.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.457268 378 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/wal
08:55:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a4881984b45041578481cb5edd541251"
08:55:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.457348 378 fs_report.cc:383] FS layout report
08:55:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:55:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/wal
08:55:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/wal
08:55:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/data/data
08:55:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:55:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:55:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:55:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:55:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:55:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:55:58.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:55:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:55:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:55:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:55:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:55:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.482810 378 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:55:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.483582 378 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:55:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.483793 378 kserver.cc:163] Server-wide thread pool size limit: 3276
08:55:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.490760 378 rpc_server.cc:307] RPC server started. Bound to: 127.0.91.254:45791
08:55:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.490767 444 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.91.254:45791 every 8 connection(s)
08:55:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.491900 378 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/data/info.pb
08:55:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.495224 445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:55:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.497270 367 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 378
08:55:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.497435 367 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-0/wal/instance
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.499159 367 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/wal
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/data
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/logs
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/data/info.pb
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:58.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:55:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:55:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:55:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:55:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.253:46043
08:55:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.253
08:55:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:55:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:55:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43845/jwks.json with env {}
08:55:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.499466 445 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.514698 445 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.515547 396 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.91.253:46043: connect: Connection refused (error 111)
08:55:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.516441 445 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.253:46043: Network error: Client connection negotiation failed: client connection to 127.0.91.253:46043: connect: Connection refused (error 111)
08:55:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.565696 445 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } attempt: 1
08:55:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.567726 445 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.253:46043: Network error: Client connection negotiation failed: client connection to 127.0.91.253:46043: connect: Connection refused (error 111)
08:55:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.613320 447 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:55:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.613575 447 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:55:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.613611 447 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:55:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.617743 447 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:55:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.617794 447 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:55:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.617812 447 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:55:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.617827 447 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:55:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.617844 447 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:55:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.617857 447 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43845/jwks.json
08:55:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.622493 447 master_runner.cc:386] Master server non-default flags:
08:55:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/data
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/wal
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.253:46043
08:55:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43845/jwks.json
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/data/info.pb
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.253
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.447
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/logs
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:55:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:55:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.623579 447 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:55:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.625142 447 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:55:58.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.633322 458 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.633319 455 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.633369 447 server_base.cc:1034] running on GCE node
08:55:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.633322 456 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.634392 447 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:55:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.635495 447 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:55:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.635911 445 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } attempt: 2
08:55:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.636669 447 hybrid_clock.cc:648] HybridClock initialized: now 1781945758636645 us; error 41 us; skew 500 ppm
08:55:58.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.637620 445 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.253:46043: Network error: Client connection negotiation failed: client connection to 127.0.91.253:46043: connect: Connection refused (error 111)
08:55:58.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.639114 447 webserver.cc:460] Webserver started at http://127.0.91.253:39321/ using document root <none> and password file <none>
08:55:58.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.639784 447 fs_manager.cc:359] Metadata directory not provided
08:55:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.639873 447 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:55:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.640118 447 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:55:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.642082 447 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/data/instance:
08:55:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f8570b482a544f0bb91da17f7dfdcb7"
08:55:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.642714 447 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/wal/instance:
08:55:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f8570b482a544f0bb91da17f7dfdcb7"
08:55:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.646669 447 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.003s sys 0.003s
08:55:58.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.649178 464 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:55:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.650480 447 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
08:55:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.650621 447 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/wal
08:55:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f8570b482a544f0bb91da17f7dfdcb7"
08:55:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.650743 447 fs_report.cc:383] FS layout report
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/wal
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/wal
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/data/data
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:55:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:55:58.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:55:58.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:55:58.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:55:58.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.671322 447 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:55:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.672204 447 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:55:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.672447 447 kserver.cc:163] Server-wide thread pool size limit: 3276
08:55:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.679865 447 rpc_server.cc:307] RPC server started. Bound to: 127.0.91.253:46043
08:55:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.679898 516 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.91.253:46043 every 8 connection(s)
08:55:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.680878 447 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/data/info.pb
08:55:58.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.684211 517 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:55:58.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.685580 367 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 447
08:55:58.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.685681 367 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-1/wal/instance
08:55:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.686379 367 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/wal
08:55:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/data
08:55:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:55:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/logs
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/data/info.pb
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:55:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.252:42963
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.252
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43845/jwks.json with env {}
08:55:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.688122 517 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.698478 517 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.705185 517 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.706034 468 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.91.252:42963: connect: Connection refused (error 111)
08:55:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.706871 517 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.252:42963: Network error: Client connection negotiation failed: client connection to 127.0.91.252:42963: connect: Connection refused (error 111)
08:55:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.728816 445 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } attempt: 3
08:55:58.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.737162 445 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.738656 445 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.252:42963: Network error: Client connection negotiation failed: client connection to 127.0.91.252:42963: connect: Connection refused (error 111)
08:55:58.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.756134 517 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } attempt: 1
08:55:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.757737 517 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.252:42963: Network error: Client connection negotiation failed: client connection to 127.0.91.252:42963: connect: Connection refused (error 111)
08:55:58.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.769832 445 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } attempt: 1
08:55:58.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.771509 445 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.252:42963: Network error: Client connection negotiation failed: client connection to 127.0.91.252:42963: connect: Connection refused (error 111)
08:55:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.804773 519 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:55:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.805043 519 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:55:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.805080 519 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:55:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.809109 519 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:55:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.809166 519 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:55:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.809185 519 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:55:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.809201 519 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:55:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.809217 519 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:55:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.809231 519 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43845/jwks.json
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.816018 519 master_runner.cc:386] Master server non-default flags:
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/data
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/wal
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.252:42963
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:55:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43845/jwks.json
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/data/info.pb
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.252
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.519
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/logs
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:55:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:55:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.817402 519 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:55:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.819100 519 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:55:58.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.825924 517 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } attempt: 2
08:55:58.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.827309 517 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.252:42963: Network error: Client connection negotiation failed: client connection to 127.0.91.252:42963: connect: Connection refused (error 111)
08:55:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.829414 528 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.829409 527 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.829385 530 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.829747 519 server_base.cc:1034] running on GCE node
08:55:58.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.830492 519 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:55:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.831511 519 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:55:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.832708 519 hybrid_clock.cc:648] HybridClock initialized: now 1781945758832676 us; error 51 us; skew 500 ppm
08:55:58.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.834436 519 webserver.cc:460] Webserver started at http://127.0.91.252:42095/ using document root <none> and password file <none>
08:55:58.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.835054 519 fs_manager.cc:359] Metadata directory not provided
08:55:58.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.835127 519 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:55:58.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.835350 519 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:55:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.837831 519 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/data/instance:
08:55:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0614714185e4ed282b35bde9e250d8c"
08:55:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.838500 519 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/wal/instance:
08:55:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0614714185e4ed282b35bde9e250d8c"
08:55:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.843081 519 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.002s
08:55:58.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.845928 536 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:55:58.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.846722 445 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } attempt: 2
08:55:58.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.847440 519 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.847779 519 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/wal
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0614714185e4ed282b35bde9e250d8c"
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:58 on dist-test-slave-5mp7"
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.848238 445 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.91.252:42963: Network error: Client connection negotiation failed: client connection to 127.0.91.252:42963: connect: Connection refused (error 111)
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.848261 519 fs_report.cc:383] FS layout report
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/wal
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/wal
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/data/data
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:55:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:55:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:55:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:55:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:55:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:55:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:55:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:55:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:55:58.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.864717 519 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:55:58.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.865459 519 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:55:58.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.865607 519 kserver.cc:163] Server-wide thread pool size limit: 3276
08:55:58.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.872937 519 rpc_server.cc:307] RPC server started. Bound to: 127.0.91.252:42963
08:55:58.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.872951 588 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.91.252:42963 every 8 connection(s)
08:55:58.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.873876 519 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/data/info.pb
08:55:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.876968 589 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:55:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.880244 589 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.883450 367 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 519
08:55:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.883529 367 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/master-2/wal/instance
08:55:58.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.890560 589 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.895143 589 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } has no permanent_uuid. Determining permanent_uuid...
08:55:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.904330 589 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c: Bootstrap starting.
08:55:58.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.906766 589 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c: Neither blocks nor log segments found. Creating new log.
08:55:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.907533 589 log.cc:824] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c: Log is configured to *not* fsync() on all Append() calls
08:55:58.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.909811 589 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c: No bootstrap required, opened a new log
08:55:58.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.913344 589 raft_consensus.cc:348] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:58.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.913538 589 raft_consensus.cc:374] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:55:58.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.913571 589 raft_consensus.cc:729] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0614714185e4ed282b35bde9e250d8c, State: Initialized, Role: FOLLOWER
08:55:58.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.913978 589 consensus_queue.cc:260] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:58.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.914398 598 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:58.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.914520 598 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [sys.catalog]: This master's current role is: FOLLOWER
08:55:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.915141 589 sys_catalog.cc:564] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [sys.catalog]: configured and running, proceeding with master startup.
08:55:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.918515 517 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } attempt: 3
08:55:58.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.920174 609 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:55:58.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.920258 609 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:55:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.919024 519 master_runner.cc:186] Error getting master registration for 127.0.91.254:45791: OK, instance_id { permanent_uuid: "a4881984b45041578481cb5edd541251" instance_seqno: 1781945758488975 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:55:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.921926 519 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:55:58.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.925673 517 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7: Bootstrap starting.
08:55:58.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.927904 517 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7: Neither blocks nor log segments found. Creating new log.
08:55:58.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.928840 517 log.cc:824] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7: Log is configured to *not* fsync() on all Append() calls
08:55:58.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.930938 517 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7: No bootstrap required, opened a new log
08:55:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.933844 517 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.934051 517 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:55:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.934106 517 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2f8570b482a544f0bb91da17f7dfdcb7, State: Initialized, Role: FOLLOWER
08:55:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.934556 517 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.935032 612 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.935158 612 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [sys.catalog]: This master's current role is: FOLLOWER
08:55:58.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.935961 517 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [sys.catalog]: configured and running, proceeding with master startup.
08:55:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.940775 623 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:55:58.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.941035 623 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:55:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.939815 447 master_runner.cc:186] Error getting master registration for 127.0.91.254:45791: OK, instance_id { permanent_uuid: "a4881984b45041578481cb5edd541251" instance_seqno: 1781945758488975 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:55:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.943493 447 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:55:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.947402 445 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } attempt: 3
08:55:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.955034 445 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251: Bootstrap starting.
08:55:58.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.958178 445 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251: Neither blocks nor log segments found. Creating new log.
08:55:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.959424 445 log.cc:824] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251: Log is configured to *not* fsync() on all Append() calls
08:55:58.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.961283 445 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251: No bootstrap required, opened a new log
08:55:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.965325 445 raft_consensus.cc:348] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.965546 445 raft_consensus.cc:374] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:55:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.965602 445 raft_consensus.cc:729] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a4881984b45041578481cb5edd541251, State: Initialized, Role: FOLLOWER
08:55:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.966104 445 consensus_queue.cc:260] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:58.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.967649 445 sys_catalog.cc:564] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: configured and running, proceeding with master startup.
08:55:58.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.970767 626 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.973917 637 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:55:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.976531 378 master_runner.cc:263] No leader master found from master a4881984b45041578481cb5edd541251
08:55:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.979740 626 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: This master's current role is: FOLLOWER
08:55:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:58.979789 378 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:55:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.979763 637 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:55:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:58.985268 398 tablet.cc:2369] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:55:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.051314 626 raft_consensus.cc:482] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:55:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.051927 626 raft_consensus.cc:504] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.056056 491 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a4881984b45041578481cb5edd541251" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" is_pre_election: true
08:55:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.056413 491 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a4881984b45041578481cb5edd541251 in term 0.
08:55:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.056865 396 leader_election.cc:302] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2f8570b482a544f0bb91da17f7dfdcb7, a4881984b45041578481cb5edd541251; no voters:
08:55:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.058292 563 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a4881984b45041578481cb5edd541251" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0614714185e4ed282b35bde9e250d8c" is_pre_election: true
08:55:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.058604 563 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a4881984b45041578481cb5edd541251 in term 0.
08:55:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.059587 626 leader_election.cc:288] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2f8570b482a544f0bb91da17f7dfdcb7 (127.0.91.253:46043), a0614714185e4ed282b35bde9e250d8c (127.0.91.252:42963)
08:55:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.060058 626 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:55:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.060307 626 raft_consensus.cc:482] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:55:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.060520 626 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 0 FOLLOWER]: Advancing to term 1
08:55:59.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.062561 626 raft_consensus.cc:504] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.063433 491 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a4881984b45041578481cb5edd541251" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2f8570b482a544f0bb91da17f7dfdcb7"
08:55:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.063560 491 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [term 0 FOLLOWER]: Advancing to term 1
08:55:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.064754 491 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a4881984b45041578481cb5edd541251 in term 1.
08:55:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.067420 396 leader_election.cc:302] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2f8570b482a544f0bb91da17f7dfdcb7, a4881984b45041578481cb5edd541251; no voters:
08:55:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.068451 563 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a4881984b45041578481cb5edd541251" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0614714185e4ed282b35bde9e250d8c"
08:55:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.068569 563 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [term 0 FOLLOWER]: Advancing to term 1
08:55:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.069695 563 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a4881984b45041578481cb5edd541251 in term 1.
08:55:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.070353 626 leader_election.cc:288] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [CANDIDATE]: Term 1 election: Requested vote from peers 2f8570b482a544f0bb91da17f7dfdcb7 (127.0.91.253:46043), a0614714185e4ed282b35bde9e250d8c (127.0.91.252:42963)
08:55:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.070979 626 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 1 FOLLOWER]: Leader election won for term 1
08:55:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.071210 626 raft_consensus.cc:686] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [term 1 LEADER]: Becoming Leader. State: Replica: a4881984b45041578481cb5edd541251, State: Running, Role: LEADER
08:55:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.072081 626 consensus_queue.cc:237] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } }
08:55:59.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.074967 639 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a4881984b45041578481cb5edd541251. Latest consensus state: current_term: 1 leader_uuid: "a4881984b45041578481cb5edd541251" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:59.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.076787 639 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: This master's current role is: LEADER
08:55:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.083391 641 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
08:55:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.084909 641 catalog_manager.cc:1484] Initializing Kudu cluster ID...
08:55:59.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.093439 563 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [term 1 FOLLOWER]: Refusing update from remote peer a4881984b45041578481cb5edd541251: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:55:59.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.094378 639 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:55:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.098845 598 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [sys.catalog]: SysCatalogTable state changed. Reason: New leader a4881984b45041578481cb5edd541251. Latest consensus state: current_term: 1 leader_uuid: "a4881984b45041578481cb5edd541251" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.098989 598 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [sys.catalog]: This master's current role is: FOLLOWER
08:55:59.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.100310 491 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [term 1 FOLLOWER]: Refusing update from remote peer a4881984b45041578481cb5edd541251: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:55:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.101286 626 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:55:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.103107 642 mvcc.cc:205] Tried to move back new op lower bound from 7298849829239246848 to 7298849829165162496. Current Snapshot: MvccSnapshot[applied={T|T < 7298849829239246848}]
08:55:59.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.105160 641 catalog_manager.cc:1347] Generated new cluster ID: ac171d235b014f42a21646129860c84e
08:55:59.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.105237 641 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
08:55:59.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.108068 612 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a4881984b45041578481cb5edd541251. Latest consensus state: current_term: 1 leader_uuid: "a4881984b45041578481cb5edd541251" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:59.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.108196 612 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [sys.catalog]: This master's current role is: FOLLOWER
08:55:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.108465 598 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a4881984b45041578481cb5edd541251" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.108590 598 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c [sys.catalog]: This master's current role is: FOLLOWER
08:55:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.109318 626 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a4881984b45041578481cb5edd541251" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.109441 626 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: This master's current role is: LEADER
08:55:59.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.113788 639 sys_catalog.cc:455] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a4881984b45041578481cb5edd541251" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:59.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.113910 639 sys_catalog.cc:458] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251 [sys.catalog]: This master's current role is: LEADER
08:55:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.115011 612 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a4881984b45041578481cb5edd541251" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a4881984b45041578481cb5edd541251" member_type: VOTER last_known_addr { host: "127.0.91.254" port: 45791 } } peers { permanent_uuid: "2f8570b482a544f0bb91da17f7dfdcb7" member_type: VOTER last_known_addr { host: "127.0.91.253" port: 46043 } } peers { permanent_uuid: "a0614714185e4ed282b35bde9e250d8c" member_type: VOTER last_known_addr { host: "127.0.91.252" port: 42963 } } }
08:55:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.115113 612 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7 [sys.catalog]: This master's current role is: FOLLOWER
08:55:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.116946 643 mvcc.cc:205] Tried to move back new op lower bound from 7298849829239246848 to 7298849829165162496. Current Snapshot: MvccSnapshot[applied={T|T < 7298849829239246848 or (T in {7298849829239246848})}]
08:55:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.117579 647 mvcc.cc:205] Tried to move back new op lower bound from 7298849829239246848 to 7298849829165162496. Current Snapshot: MvccSnapshot[applied={T|T < 7298849829239246848}]
08:55:59.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.121863 641 catalog_manager.cc:1370] Generated new certificate authority record
08:55:59.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.123343 641 catalog_manager.cc:1504] Loading token signing keys...
08:55:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.139199 641 catalog_manager.cc:5868] T 00000000000000000000000000000000 P a4881984b45041578481cb5edd541251: Generated new TSK 0
08:55:59.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.140411 641 catalog_manager.cc:1514] Initializing in-progress tserver states...
08:55:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.154307 367 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/wal
08:55:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/data
08:55:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:55:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:55:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:55:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/logs
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/data/info.pb
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.193:0
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.91.193
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.193
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:55:59.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.269806 650 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:55:59.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.270140 650 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:55:59.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.270197 650 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:55:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.274537 650 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:55:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.274679 650 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.91.193
08:55:59.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.279610 650 tablet_server_runner.cc:78] Tablet server non-default flags:
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/data
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/wal
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.193:0
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/data/info.pb
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.193
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.650
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.91.193
08:55:59.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/logs
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:55:59.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.281052 650 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:55:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.282552 650 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:55:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.292498 656 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.292480 658 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.292482 655 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.293455 650 server_base.cc:1034] running on GCE node
08:55:59.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.293895 650 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:55:59.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.294549 650 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:55:59.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.295693 650 hybrid_clock.cc:648] HybridClock initialized: now 1781945759295667 us; error 35 us; skew 500 ppm
08:55:59.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.297696 650 webserver.cc:460] Webserver started at http://127.0.91.193:33103/ using document root <none> and password file <none>
08:55:59.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.298316 650 fs_manager.cc:359] Metadata directory not provided
08:55:59.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.298404 650 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:55:59.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.298647 650 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:55:59.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.300647 650 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/data/instance:
08:55:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ccfdc5a6b434a17a4ff781d14cf3071"
08:55:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.301230 650 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/wal/instance:
08:55:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ccfdc5a6b434a17a4ff781d14cf3071"
08:55:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.304900 650 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.003s sys 0.001s
08:55:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.307358 664 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:55:59.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.308581 650 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
08:55:59.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.308710 650 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/wal
08:55:59.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ccfdc5a6b434a17a4ff781d14cf3071"
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.308851 650 fs_report.cc:383] FS layout report
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/wal
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/wal
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/data/data
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:55:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:55:59.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.331848 650 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:55:59.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.332679 650 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:55:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.332892 650 kserver.cc:163] Server-wide thread pool size limit: 3276
08:55:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.333514 650 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:55:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.334540 650 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:55:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.334607 650 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:55:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.334661 650 ts_tablet_manager.cc:610] Registered 0 tablets
08:55:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.334684 650 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:55:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.344992 650 rpc_server.cc:307] RPC server started. Bound to: 127.0.91.193:38617
08:55:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.345008 777 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.91.193:38617 every 8 connection(s)
08:55:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.346081 650 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/data/info.pb
08:55:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.350538 367 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 650
08:55:59.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.350657 367 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-0/wal/instance
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.353657 367 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/wal
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/data
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/logs
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/data/info.pb
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:55:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.194:0
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.91.194
08:55:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.194
08:55:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360024 778 heartbeater.cc:347] Connected to a master server at 127.0.91.252:42963
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360106 780 heartbeater.cc:347] Connected to a master server at 127.0.91.253:46043
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360249 779 heartbeater.cc:347] Connected to a master server at 127.0.91.254:45791
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360280 778 heartbeater.cc:464] Registering TS with master...
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360280 780 heartbeater.cc:464] Registering TS with master...
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360344 779 heartbeater.cc:464] Registering TS with master...
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360968 780 heartbeater.cc:511] Master 127.0.91.253:46043 requested a full tablet report, sending...
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360982 779 heartbeater.cc:511] Master 127.0.91.254:45791 requested a full tablet report, sending...
08:55:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.360968 778 heartbeater.cc:511] Master 127.0.91.252:42963 requested a full tablet report, sending...
08:55:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.362726 481 ts_manager.cc:194] Registered new tserver with Master: 5ccfdc5a6b434a17a4ff781d14cf3071 (127.0.91.193:38617)
08:55:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.362848 408 ts_manager.cc:194] Registered new tserver with Master: 5ccfdc5a6b434a17a4ff781d14cf3071 (127.0.91.193:38617)
08:55:59.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.364244 553 ts_manager.cc:194] Registered new tserver with Master: 5ccfdc5a6b434a17a4ff781d14cf3071 (127.0.91.193:38617)
08:55:59.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.364573 408 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.91.193:59691
08:55:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.478719 785 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:55:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.479017 785 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:55:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.479066 785 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:55:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.483516 785 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:55:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.483677 785 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.91.194
08:55:59.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.488667 785 tablet_server_runner.cc:78] Tablet server non-default flags:
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/data
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/wal
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.194:0
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/data/info.pb
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.194
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.785
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.91.194
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/logs
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:55:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:55:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:55:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:55:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:55:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:55:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:55:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:55:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.490116 785 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:55:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.491842 785 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:55:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.501310 790 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.501454 791 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.501310 793 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.501904 785 server_base.cc:1034] running on GCE node
08:55:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.502858 785 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:55:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.503578 785 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:55:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.504773 785 hybrid_clock.cc:648] HybridClock initialized: now 1781945759504740 us; error 54 us; skew 500 ppm
08:55:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.507066 785 webserver.cc:460] Webserver started at http://127.0.91.194:33037/ using document root <none> and password file <none>
08:55:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.507783 785 fs_manager.cc:359] Metadata directory not provided
08:55:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.507874 785 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:55:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.508121 785 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:55:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.510237 785 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/data/instance:
08:55:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d20b1f0b4d804b04bd4131eeb82ca8e8"
08:55:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.510850 785 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/wal/instance:
08:55:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d20b1f0b4d804b04bd4131eeb82ca8e8"
08:55:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.514865 785 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.001s
08:55:59.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.517474 799 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:55:59.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.518751 785 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.518904 785 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/wal
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d20b1f0b4d804b04bd4131eeb82ca8e8"
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.519019 785 fs_report.cc:383] FS layout report
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/wal
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/wal
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/data/data
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:55:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:55:59.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.529199 785 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:55:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.529892 785 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:55:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.530102 785 kserver.cc:163] Server-wide thread pool size limit: 3276
08:55:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.531082 785 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:55:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.532334 785 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:55:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.532402 785 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:55:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.532456 785 ts_tablet_manager.cc:610] Registered 0 tablets
08:55:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.532480 785 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:55:59.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.542258 785 rpc_server.cc:307] RPC server started. Bound to: 127.0.91.194:45927
08:55:59.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.542308 912 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.91.194:45927 every 8 connection(s)
08:55:59.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.543258 785 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/data/info.pb
08:55:59.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.553107 367 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 785
08:55:59.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.553220 367 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-1/wal/instance
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.555724 367 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/wal
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/data
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/logs
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/data/info.pb
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.195:0
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.91.195
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.195
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.556447 915 heartbeater.cc:347] Connected to a master server at 127.0.91.253:46043
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.556675 915 heartbeater.cc:464] Registering TS with master...
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.556934 913 heartbeater.cc:347] Connected to a master server at 127.0.91.252:42963
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.557030 913 heartbeater.cc:464] Registering TS with master...
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.557228 913 heartbeater.cc:511] Master 127.0.91.252:42963 requested a full tablet report, sending...
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.557228 915 heartbeater.cc:511] Master 127.0.91.253:46043 requested a full tablet report, sending...
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.558279 481 ts_manager.cc:194] Registered new tserver with Master: d20b1f0b4d804b04bd4131eeb82ca8e8 (127.0.91.194:45927)
08:55:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.558308 553 ts_manager.cc:194] Registered new tserver with Master: d20b1f0b4d804b04bd4131eeb82ca8e8 (127.0.91.194:45927)
08:55:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.559325 914 heartbeater.cc:347] Connected to a master server at 127.0.91.254:45791
08:55:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.559407 914 heartbeater.cc:464] Registering TS with master...
08:55:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.559526 914 heartbeater.cc:511] Master 127.0.91.254:45791 requested a full tablet report, sending...
08:55:59.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.560082 408 ts_manager.cc:194] Registered new tserver with Master: d20b1f0b4d804b04bd4131eeb82ca8e8 (127.0.91.194:45927)
08:55:59.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.560760 408 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.91.194:47727
08:55:59.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.670477 920 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:55:59.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.670847 920 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:55:59.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.670902 920 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:55:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.675434 920 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:55:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.675559 920 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.91.195
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.680335 920 tablet_server_runner.cc:78] Tablet server non-default flags:
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.91.212:42007
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/data
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/wal
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.91.195:0
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:55:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/data/info.pb
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.91.195
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.91.254:45791,127.0.91.253:46043,127.0.91.252:42963
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.920
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.91.195
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/logs
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:55:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.681581 920 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:55:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.682945 920 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:55:59.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.691926 926 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.691947 925 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:55:59.691996 928 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:55:59.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.692173 920 server_base.cc:1034] running on GCE node
08:55:59.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.692788 920 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:55:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.693475 920 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:55:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.694645 920 hybrid_clock.cc:648] HybridClock initialized: now 1781945759694621 us; error 50 us; skew 500 ppm
08:55:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.696815 920 webserver.cc:460] Webserver started at http://127.0.91.195:39023/ using document root <none> and password file <none>
08:55:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.697329 920 fs_manager.cc:359] Metadata directory not provided
08:55:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.697427 920 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:55:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.697646 920 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:55:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.699522 920 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/data/instance:
08:55:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "686277600b534533a403edaa468fdf05"
08:55:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.700078 920 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/wal/instance:
08:55:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "686277600b534533a403edaa468fdf05"
08:55:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.703440 920 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.001s
08:55:59.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.705672 934 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:55:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.706825 920 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.706944 920 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/wal
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "686277600b534533a403edaa468fdf05"
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:55:59 on dist-test-slave-5mp7"
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.707070 920 fs_report.cc:383] FS layout report
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/wal
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/wal
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/data/data
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:55:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:55:59.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.738198 920 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:55:59.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.739001 920 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:55:59.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.739702 920 kserver.cc:163] Server-wide thread pool size limit: 3276
08:55:59.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.740341 920 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:55:59.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.741366 920 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:55:59.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.741436 920 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:55:59.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.741493 920 ts_tablet_manager.cc:610] Registered 0 tablets
08:55:59.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.741515 920 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:55:59.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.750624 920 rpc_server.cc:307] RPC server started. Bound to: 127.0.91.195:44541
08:55:59.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.750682 1047 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.91.195:44541 every 8 connection(s)
08:55:59.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.751590 920 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/data/info.pb
08:55:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.753224 367 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 920
08:55:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.753324 367 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster8335217257930081453/ts-2/wal/instance
08:55:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.763872 1050 heartbeater.cc:347] Connected to a master server at 127.0.91.253:46043
08:55:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.763880 1049 heartbeater.cc:347] Connected to a master server at 127.0.91.254:45791
08:55:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.763974 1048 heartbeater.cc:347] Connected to a master server at 127.0.91.252:42963
08:55:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.764125 1050 heartbeater.cc:464] Registering TS with master...
08:55:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.764125 1048 heartbeater.cc:464] Registering TS with master...
08:55:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.764125 1049 heartbeater.cc:464] Registering TS with master...
08:55:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.764724 1049 heartbeater.cc:511] Master 127.0.91.254:45791 requested a full tablet report, sending...
08:55:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.764704 1048 heartbeater.cc:511] Master 127.0.91.252:42963 requested a full tablet report, sending...
08:55:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.764715 1050 heartbeater.cc:511] Master 127.0.91.253:46043 requested a full tablet report, sending...
08:55:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.765825 408 ts_manager.cc:194] Registered new tserver with Master: 686277600b534533a403edaa468fdf05 (127.0.91.195:44541)
08:55:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.765924 481 ts_manager.cc:194] Registered new tserver with Master: 686277600b534533a403edaa468fdf05 (127.0.91.195:44541)
08:55:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.765933 553 ts_manager.cc:194] Registered new tserver with Master: 686277600b534533a403edaa468fdf05 (127.0.91.195:44541)
08:55:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.766597 408 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.91.195:54345
08:55:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.773821 367 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
08:55:59.774 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:55:59.775 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:55:59.785 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.91.254"
port: 45791
}
bound_http_address {
host: "127.0.91.254"
port: 33661
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.91.253"
port: 46043
}
bound_http_address {
host: "127.0.91.253"
port: 39321
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.91.252"
port: 42963
}
bound_http_address {
host: "127.0.91.252"
port: 42095
}
}
}
08:55:59.791 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:55:59.794 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.91.193"
port: 38617
}
bound_http_address {
host: "127.0.91.193"
port: 33103
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.91.194"
port: 45927
}
bound_http_address {
host: "127.0.91.194"
port: 33037
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.91.195"
port: 44541
}
bound_http_address {
host: "127.0.91.195"
port: 39023
}
}
}
08:55:59.795 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:55:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.921676 609 catalog_manager.cc:1259] Loaded cluster ID: ac171d235b014f42a21646129860c84e
08:55:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.921775 609 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c: loading cluster ID for follower catalog manager: success
08:55:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.923882 609 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c: acquiring CA information for follower catalog manager: success
08:55:59.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.925213 609 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a0614714185e4ed282b35bde9e250d8c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:55:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.942454 623 catalog_manager.cc:1259] Loaded cluster ID: ac171d235b014f42a21646129860c84e
08:55:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.942545 623 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7: loading cluster ID for follower catalog manager: success
08:55:59.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.944393 623 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7: acquiring CA information for follower catalog manager: success
08:55:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:55:59.945577 623 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2f8570b482a544f0bb91da17f7dfdcb7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:55:59.986 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:56:00.107 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task5KApzg/test-tmp
08:56:00.108 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
08:56:00.108 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task5KApzg/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
08:56:00.112 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:56:00.228 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:00.228 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:56:00.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:00.229404 1056 mini_oidc.cc:152] Starting JWKS server
08:56:00.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:00.231544 1056 webserver.cc:460] Webserver started at http://127.0.0.1:40601/ using document root <none> and password file <none>
08:56:00.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:00.231742 1056 mini_oidc.cc:189] Starting OIDC Discovery server
08:56:00.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:00.232038 1056 webserver.cc:460] Webserver started at http://127.0.0.1:33299/ using document root <none> and password file <none>
08:56:00.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:56:00.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/principal' for realm 'KRBTEST.COM',
08:56:00.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:56:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:00 dist-test-slave-5mp7 krb5kdc[1067](info): setting up network...
08:56:00.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:56:00.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:00 dist-test-slave-5mp7 krb5kdc[1067](info): set up 2 sockets
08:56:00.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:00 dist-test-slave-5mp7 krb5kdc[1067](info): commencing operation
08:56:00.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:56:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:00.367219 779 heartbeater.cc:503] Master 127.0.91.254:45791 was elected leader, sending a full tablet report...
08:56:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:00.562935 914 heartbeater.cc:503] Master 127.0.91.254:45791 was elected leader, sending a full tablet report...
08:56:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:00.768318 1049 heartbeater.cc:503] Master 127.0.91.254:45791 was elected leader, sending a full tablet report...
08:56:01.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.285035 1056 mini_kdc.cc:121] Time spent starting KDC: real 1.045s user 0.002s sys 0.005s
08:56:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:56:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:56:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:56:01.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:56:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:56:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:56:01.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:56:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:56:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:56:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:56:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/test-user.keytab.
08:56:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/test-user.keytab.
08:56:01.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:01 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:01.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:56:01.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T08:56:01Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:56:01.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T08:56:01Z Disabled control of system clock
08:56:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.8.62@KRBTEST.COM; defaulting to no policy
08:56:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:56:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.8.62@KRBTEST.COM" created.
08:56:01.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:56:01.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.8.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.62.keytab.
08:56:01.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.8.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.62.keytab.
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.421036 1056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/logs
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/info.pb
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:56:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.62:41091
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.62
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.62.keytab
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.62
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:56:01.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.540024 1080 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:56:01.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.540360 1080 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:56:01.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.540424 1080 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:56:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.544615 1080 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:56:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.544695 1080 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:56:01.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.544735 1080 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:56:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.544757 1080 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:56:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.544780 1080 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:56:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.544802 1080 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:56:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.544821 1080 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40601/jwks.json
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.550284 1080 master_runner.cc:386] Master server non-default flags:
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.62.keytab
08:56:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.62
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.62:41091
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/info.pb
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.62
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1080
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/logs
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:56:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:56:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:56:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:56:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:56:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:56:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:56:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:56:01.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.551779 1080 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:56:01.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.552978 1080 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:56:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.562464 1085 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.562526 1086 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.562759 1080 server_base.cc:1034] running on GCE node
08:56:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.562479 1088 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.563582 1080 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:56:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.564875 1080 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:56:01.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.566116 1080 hybrid_clock.cc:648] HybridClock initialized: now 1781945761566094 us; error 41 us; skew 500 ppm
08:56:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:01 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:01.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.569259 1080 init.cc:377] Logged in from keytab as kudu/127.1.8.62@KRBTEST.COM (short username kudu)
08:56:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.570323 1080 webserver.cc:460] Webserver started at http://127.1.8.62:44555/ using document root <none> and password file <none>
08:56:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.570906 1080 fs_manager.cc:359] Metadata directory not provided
08:56:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.570996 1080 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:56:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.571230 1080 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:56:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.573239 1080 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/instance:
08:56:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c35c491c2cbb47f29eed656fbf5ccbce"
08:56:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:01 on dist-test-slave-5mp7"
08:56:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.573879 1080 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal/instance:
08:56:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c35c491c2cbb47f29eed656fbf5ccbce"
08:56:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:01 on dist-test-slave-5mp7"
08:56:01.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.578048 1080 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
08:56:01.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.580415 1095 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:56:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.581650 1080 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
08:56:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.581755 1080 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c35c491c2cbb47f29eed656fbf5ccbce"
08:56:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:01 on dist-test-slave-5mp7"
08:56:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.581846 1080 fs_report.cc:383] FS layout report
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/data
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:56:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:56:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.605111 1080 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:56:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.608518 1080 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:56:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.608675 1080 kserver.cc:163] Server-wide thread pool size limit: 3276
08:56:01.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.616927 1080 rpc_server.cc:307] RPC server started. Bound to: 127.1.8.62:41091
08:56:01.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.617249 1147 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.8.62:41091 every 8 connection(s)
08:56:01.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.617823 1080 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/info.pb
08:56:01.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.620579 1148 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:56:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.624228 1148 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } has no permanent_uuid. Determining permanent_uuid...
08:56:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.628129 1056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 1080
08:56:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.628300 1056 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal/instance
08:56:01.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:01 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.62@KRBTEST.COM for kudu/127.1.8.62@KRBTEST.COM
08:56:01.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.8.61@KRBTEST.COM; defaulting to no policy
08:56:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:56:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.8.61@KRBTEST.COM" created.
08:56:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.644033 1148 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } has no permanent_uuid. Determining permanent_uuid...
08:56:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.645033 1097 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.646000 1148 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.61:40581: Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:56:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.8.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.61.keytab.
08:56:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.8.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.61.keytab.
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.658385 1056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/logs
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/info.pb
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.61:40581
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.61
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.61.keytab
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.61
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:56:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.695288 1148 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } attempt: 1
08:56:01.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.696818 1148 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.61:40581: Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:01.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.764993 1148 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } attempt: 2
08:56:01.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.766425 1148 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.61:40581: Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:01.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.776187 1155 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:56:01.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.776503 1155 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:56:01.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.776563 1155 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:56:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.780788 1155 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:56:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.780864 1155 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:56:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.780905 1155 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:56:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.780926 1155 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:56:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.780947 1155 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:56:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.780968 1155 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:56:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.780985 1155 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40601/jwks.json
08:56:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.786585 1155 master_runner.cc:386] Master server non-default flags:
08:56:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.61.keytab
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.61
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.61:40581
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/info.pb
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.61
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1155
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/logs
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:56:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.788012 1155 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:56:01.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.789085 1155 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:56:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.797453 1163 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.797461 1161 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.797462 1160 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:01.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.797816 1155 server_base.cc:1034] running on GCE node
08:56:01.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.798364 1155 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:56:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.799635 1155 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:56:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.800819 1155 hybrid_clock.cc:648] HybridClock initialized: now 1781945761800792 us; error 49 us; skew 500 ppm
08:56:01.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:01 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:01.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.803745 1155 init.cc:377] Logged in from keytab as kudu/127.1.8.61@KRBTEST.COM (short username kudu)
08:56:01.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.804571 1155 webserver.cc:460] Webserver started at http://127.1.8.61:46681/ using document root <none> and password file <none>
08:56:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.805068 1155 fs_manager.cc:359] Metadata directory not provided
08:56:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.805123 1155 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:56:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.805300 1155 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:56:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.807097 1155 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/instance:
08:56:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14fe1efbcfb94507b4b151ed7c56e1b4"
08:56:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:01 on dist-test-slave-5mp7"
08:56:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.807660 1155 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal/instance:
08:56:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14fe1efbcfb94507b4b151ed7c56e1b4"
08:56:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:01 on dist-test-slave-5mp7"
08:56:01.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.811241 1155 fs_manager.cc:691] Time spent creating directory manager: real 0.003s user 0.002s sys 0.000s
08:56:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.813723 1170 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.814972 1155 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.815116 1155 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14fe1efbcfb94507b4b151ed7c56e1b4"
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:01 on dist-test-slave-5mp7"
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.815248 1155 fs_report.cc:383] FS layout report
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/data
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:56:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:56:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:56:01.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.828514 1155 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:56:01.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.831375 1155 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:56:01.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.831568 1155 kserver.cc:163] Server-wide thread pool size limit: 3276
08:56:01.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.839658 1155 rpc_server.cc:307] RPC server started. Bound to: 127.1.8.61:40581
08:56:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.839900 1222 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.8.61:40581 every 8 connection(s)
08:56:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.840682 1155 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/info.pb
08:56:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.843568 1223 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:56:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.844070 1056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 1155
08:56:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.844178 1056 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal/instance
08:56:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.847397 1223 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } has no permanent_uuid. Determining permanent_uuid...
08:56:01.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.8.60@KRBTEST.COM; defaulting to no policy
08:56:01.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:56:01.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.8.60@KRBTEST.COM" created.
08:56:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:01 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.61@KRBTEST.COM for kudu/127.1.8.62@KRBTEST.COM
08:56:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.857601 1148 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } attempt: 3
08:56:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.864117 1223 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } has no permanent_uuid. Determining permanent_uuid...
08:56:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:01 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.62@KRBTEST.COM for kudu/127.1.8.61@KRBTEST.COM
08:56:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:01 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.61@KRBTEST.COM for kudu/127.1.8.61@KRBTEST.COM
08:56:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:56:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.8.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.60.keytab.
08:56:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.8.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.60.keytab.
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.871336 1148 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } has no permanent_uuid. Determining permanent_uuid...
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.871426 1056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/logs
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/info.pb
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.60:41503
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.60
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.60.keytab
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.60
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:56:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.872864 1148 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.60:41503: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.873950 1223 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } has no permanent_uuid. Determining permanent_uuid...
08:56:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.874682 1173 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:01.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.875677 1223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.60:41503: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.904016 1148 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } attempt: 1
08:56:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.905431 1148 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.60:41503: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:01.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.924959 1223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } attempt: 1
08:56:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.926543 1223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.60:41503: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.980614 1148 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } attempt: 2
08:56:01.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.982203 1148 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.60:41503: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.993016 1231 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:56:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.993310 1231 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:56:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.993372 1231 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:56:01.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:01.994818 1223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } attempt: 2
08:56:01.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.996331 1223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.60:41503: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.997892 1231 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:56:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.997937 1231 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:56:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.997959 1231 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:56:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.997977 1231 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:56:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.997992 1231 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:56:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.998008 1231 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:56:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:01.998023 1231 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40601/jwks.json
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.003221 1231 master_runner.cc:386] Master server non-default flags:
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:02.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.60.keytab
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.60
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.60:41503
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/info.pb
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.60
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1231
08:56:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:56:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:56:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:02.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/logs
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.004418 1231 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:56:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.005584 1231 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:56:02.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.015456 1237 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.015416 1238 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.015416 1240 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.021039 1231 server_base.cc:1034] running on GCE node
08:56:02.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.021744 1231 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:56:02.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.023114 1231 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:56:02.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.024354 1231 hybrid_clock.cc:648] HybridClock initialized: now 1781945762024312 us; error 58 us; skew 500 ppm
08:56:02.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:02.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.027222 1231 init.cc:377] Logged in from keytab as kudu/127.1.8.60@KRBTEST.COM (short username kudu)
08:56:02.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.028258 1231 webserver.cc:460] Webserver started at http://127.1.8.60:38625/ using document root <none> and password file <none>
08:56:02.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.028764 1231 fs_manager.cc:359] Metadata directory not provided
08:56:02.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.028826 1231 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:56:02.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.028991 1231 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:56:02.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.030910 1231 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/instance:
08:56:02.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "366e50900d05499fbee1850e1cd07db6"
08:56:02.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:02 on dist-test-slave-5mp7"
08:56:02.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.031399 1231 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal/instance:
08:56:02.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "366e50900d05499fbee1850e1cd07db6"
08:56:02.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:02 on dist-test-slave-5mp7"
08:56:02.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.035508 1231 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.003s sys 0.000s
08:56:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.038997 1247 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:56:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.040745 1231 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.040903 1231 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "366e50900d05499fbee1850e1cd07db6"
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:02 on dist-test-slave-5mp7"
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.041036 1231 fs_report.cc:383] FS layout report
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/data
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:56:02.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:56:02.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.079734 1231 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:56:02.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.081383 1148 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } attempt: 3
08:56:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.082863 1148 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.60:41503: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.084466 1231 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:56:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.084671 1231 kserver.cc:163] Server-wide thread pool size limit: 3276
08:56:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.087481 1223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } attempt: 3
08:56:02.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.088811 1223 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.8.60:41503: Network error: Client connection negotiation failed: client connection to 127.1.8.60:41503: connect: Connection refused (error 111)
08:56:02.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.094576 1231 rpc_server.cc:307] RPC server started. Bound to: 127.1.8.60:41503
08:56:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.095096 1299 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.8.60:41503 every 8 connection(s)
08:56:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.095733 1231 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/info.pb
08:56:02.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.099628 1300 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:56:02.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.102877 1056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 1231
08:56:02.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.103003 1056 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal/instance
08:56:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.105170 1300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } has no permanent_uuid. Determining permanent_uuid...
08:56:02.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.60@KRBTEST.COM for kudu/127.1.8.62@KRBTEST.COM
08:56:02.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.8.60@KRBTEST.COM
08:56:02.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.124460 1300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } has no permanent_uuid. Determining permanent_uuid...
08:56:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.60@KRBTEST.COM for kudu/127.1.8.61@KRBTEST.COM
08:56:02.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.8.62@KRBTEST.COM
08:56:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.136106 1300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } has no permanent_uuid. Determining permanent_uuid...
08:56:02.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.60@KRBTEST.COM for kudu/127.1.8.60@KRBTEST.COM
08:56:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.8.61@KRBTEST.COM
08:56:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.148540 1300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: Bootstrap starting.
08:56:02.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.151741 1300 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: Neither blocks nor log segments found. Creating new log.
08:56:02.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.152678 1300 log.cc:824] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: Log is configured to *not* fsync() on all Append() calls
08:56:02.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.152938 1056 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
08:56:02.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.154824 1300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: No bootstrap required, opened a new log
08:56:02.155 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:56:02.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.158514 1300 raft_consensus.cc:348] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.158771 1300 raft_consensus.cc:374] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:56:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.158830 1300 raft_consensus.cc:729] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 366e50900d05499fbee1850e1cd07db6, State: Initialized, Role: FOLLOWER
08:56:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.159404 1300 consensus_queue.cc:260] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.159981 1306 sys_catalog.cc:455] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } } }
08:56:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.160136 1306 sys_catalog.cc:458] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [sys.catalog]: This master's current role is: FOLLOWER
08:56:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.160853 1300 sys_catalog.cc:564] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [sys.catalog]: configured and running, proceeding with master startup.
08:56:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.166062 1317 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.166298 1317 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.164876 1231 master_runner.cc:186] Error getting master registration for 127.1.8.62:41091: OK, instance_id { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" instance_seqno: 1781945761614079 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:56:02.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.169071 1231 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:56:02.169 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:56:02.170 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:56:02.173 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.8.62"
port: 41091
}
bound_http_address {
host: "127.1.8.62"
port: 44555
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.8.61"
port: 40581
}
bound_http_address {
host: "127.1.8.61"
port: 46681
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.8.60"
port: 41503
}
bound_http_address {
host: "127.1.8.60"
port: 38625
}
}
}
08:56:02.173 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:56:02.174 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.174 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:56:02.176 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:56:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:56:02.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.194 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc
08:56:02.202 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:56:02.202 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc
08:56:02.203 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:56:02.207 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:56:02.209 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:56:02.209 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:56:02.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:02.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:56:02.226 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.226 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc
08:56:02.228 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:56:02.229 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.8.62:41091
08:56:02.231 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:56:02.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.231451 1056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 1080
08:56:02.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.231963 1223 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } attempt: 4
08:56:02.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.61@KRBTEST.COM for kudu/127.1.8.60@KRBTEST.COM
08:56:02.239 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.239 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.8.61:40581
08:56:02.240 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:56:02.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.240336 1056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 1155
08:56:02.250 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.250 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.8.60:41503
08:56:02.250 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.251169 1056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 1231
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.251209 1302 negotiation.cc:337] Failed RPC negotiation. Trace:
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234009 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:34354 (local address 127.1.8.60:41503)
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234123 (+ 114us) server_negotiation.cc:206] Beginning negotiation
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234128 (+ 5us) server_negotiation.cc:399] Waiting for connection header
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234141 (+ 13us) server_negotiation.cc:407] Connection header received
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234185 (+ 44us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234187 (+ 2us) server_negotiation.cc:461] Received NEGOTIATE request from client
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234231 (+ 44us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234307 (+ 76us) server_negotiation.cc:226] Negotiated authn=SASL
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.234942 (+ 635us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.235493 (+ 551us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.236360 (+ 867us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.236641 (+ 281us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
08:56:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.236756 (+ 115us) server_negotiation.cc:657] Negotiated auth-only TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
08:56:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.239072 (+ 2316us) server_negotiation.cc:365] Received SASL_INITIATE NegotiatePB request
08:56:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.239075 (+ 3us) server_negotiation.cc:881] Received SASL_INITIATE request from client
08:56:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.239078 (+ 3us) server_negotiation.cc:892] Client requested to use mechanism: GSSAPI
08:56:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.239103 (+ 25us) server_negotiation.cc:922] calling sasl_server_start()
08:56:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.241136 (+ 2033us) server_negotiation.cc:377] Sending SASL_CHALLENGE NegotiatePB response
08:56:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0620 08:56:02.250013 (+ 8877us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34354: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:56:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":49}
08:56:02.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.257 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.8.62:41091
08:56:02.258 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.258944 1056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/logs
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/info.pb
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.62:41091
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.62
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44555
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.62.keytab
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.62
08:56:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:56:02.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.374567 1323 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:56:02.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.374861 1323 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:56:02.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.374905 1323 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:56:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.379137 1323 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:56:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.379204 1323 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:56:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.379227 1323 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:56:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.379244 1323 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:56:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.379261 1323 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:56:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.379276 1323 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:56:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.379289 1323 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40601/jwks.json
08:56:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.384721 1323 master_runner.cc:386] Master server non-default flags:
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.62.keytab
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.62
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.62:41091
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/info.pb
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.62
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44555
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1323
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:56:02.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/logs
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.385780 1323 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:56:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.386726 1323 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:56:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.394855 1328 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.394855 1329 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.395027 1323 server_base.cc:1034] running on GCE node
08:56:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.394897 1331 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.395948 1323 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:56:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.397035 1323 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:56:02.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.398308 1323 hybrid_clock.cc:648] HybridClock initialized: now 1781945762398269 us; error 46 us; skew 500 ppm
08:56:02.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.401125 1323 init.cc:377] Logged in from keytab as kudu/127.1.8.62@KRBTEST.COM (short username kudu)
08:56:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.402050 1323 webserver.cc:460] Webserver started at http://127.1.8.62:44555/ using document root <none> and password file <none>
08:56:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.402627 1323 fs_manager.cc:359] Metadata directory not provided
08:56:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.402730 1323 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:56:02.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.406503 1323 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.002s sys 0.000s
08:56:02.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.408633 1338 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:56:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.409950 1323 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
08:56:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.410079 1323 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c35c491c2cbb47f29eed656fbf5ccbce"
08:56:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:01 on dist-test-slave-5mp7"
08:56:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.410663 1323 fs_report.cc:383] FS layout report
08:56:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:56:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/data
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:56:02.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:56:02.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.428540 1323 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:56:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.431777 1323 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:56:02.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.431988 1323 kserver.cc:163] Server-wide thread pool size limit: 3276
08:56:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.439677 1323 rpc_server.cc:307] RPC server started. Bound to: 127.1.8.62:41091
08:56:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.439909 1390 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.8.62:41091 every 8 connection(s)
08:56:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.440958 1323 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/data/info.pb
08:56:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.445125 1056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 1323
08:56:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.445199 1391 sys_catalog.cc:263] Verifying existing consensus state
08:56:02.445 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.445 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.8.61:40581
08:56:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 08:56:02.445587 1391 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:56:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.445736 1323 master.cc:537] Master@127.1.8.62:41091 shutting down...
08:56:02.446 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.446465 1056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/logs
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/info.pb
08:56:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.61:40581
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.61
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46681
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.61.keytab
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.61
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:56:02.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.465624 1323 master.cc:552] Master@127.1.8.62:41091 shutdown complete.
08:56:02.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:56:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.575263 1393 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:56:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.575549 1393 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:56:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.575619 1393 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:56:02.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.579912 1393 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:56:02.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.579974 1393 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:56:02.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.579996 1393 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:56:02.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.580014 1393 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:56:02.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.580030 1393 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:56:02.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.580046 1393 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:56:02.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.580060 1393 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40601/jwks.json
08:56:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.585460 1393 master_runner.cc:386] Master server non-default flags:
08:56:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data
08:56:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.61.keytab
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.61
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.61:40581
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/info.pb
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.61
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46681
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1393
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/logs
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:56:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.586666 1393 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:56:02.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.588002 1393 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:56:02.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.597586 1399 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.597600 1401 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.597596 1398 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.597739 1393 server_base.cc:1034] running on GCE node
08:56:02.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.598501 1393 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:56:02.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.599568 1393 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:56:02.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.600790 1393 hybrid_clock.cc:648] HybridClock initialized: now 1781945762600757 us; error 51 us; skew 500 ppm
08:56:02.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:02.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.603871 1393 init.cc:377] Logged in from keytab as kudu/127.1.8.61@KRBTEST.COM (short username kudu)
08:56:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.605044 1393 webserver.cc:460] Webserver started at http://127.1.8.61:46681/ using document root <none> and password file <none>
08:56:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.605804 1393 fs_manager.cc:359] Metadata directory not provided
08:56:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.605883 1393 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:56:02.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.609807 1393 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.004s sys 0.000s
08:56:02.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.612043 1408 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:56:02.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.613443 1393 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.613598 1393 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14fe1efbcfb94507b4b151ed7c56e1b4"
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:01 on dist-test-slave-5mp7"
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.614187 1393 fs_report.cc:383] FS layout report
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/data
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:56:02.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:56:02.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.640699 1393 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:56:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.644038 1393 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:56:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.644241 1393 kserver.cc:163] Server-wide thread pool size limit: 3276
08:56:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.651794 1393 rpc_server.cc:307] RPC server started. Bound to: 127.1.8.61:40581
08:56:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.652030 1460 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.8.61:40581 every 8 connection(s)
08:56:02.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.652978 1393 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/data/info.pb
08:56:02.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.653385 1056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 1393
08:56:02.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.654 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.8.60:41503
08:56:02.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.655277 1056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task5KApzg/build/debug/bin/kudu
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/logs
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/info.pb
08:56:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.60:41503
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.60
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38625
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.60.keytab
08:56:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.60
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:56:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.658090 1461 sys_catalog.cc:263] Verifying existing consensus state
08:56:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 08:56:02.658509 1461 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:56:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.658700 1393 master.cc:537] Master@127.1.8.61:40581 shutting down...
08:56:02.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.675132 1393 master.cc:552] Master@127.1.8.61:40581 shutdown complete.
08:56:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:56:02.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.775415 1463 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:56:02.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.775765 1463 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:56:02.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.775815 1463 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:56:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.780194 1463 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:56:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.780258 1463 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:56:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.780282 1463 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:56:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.780299 1463 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:56:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.780315 1463 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:56:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.780331 1463 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:56:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.780345 1463 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40601/jwks.json
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.785873 1463 master_runner.cc:386] Master server non-default flags:
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.8.20:38233
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/krb5kdc/kudu_127.1.8.60.keytab
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.8.60
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.8.60:41503
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40601/jwks.json
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/info.pb
08:56:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.8.60
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38625
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.1463
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/logs
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 08:13:29 UTC on ccycloud.quasar-srxcxo.root.comops.site
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457054
08:56:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.786993 1463 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:56:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.788043 1463 file_cache.cc:493] Constructed file cache file cache with capacity 419430
08:56:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.796463 1471 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.796473 1468 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.796559 1463 server_base.cc:1034] running on GCE node
08:56:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.796463 1469 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:56:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.797544 1463 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:56:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.798714 1463 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:56:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.799911 1463 hybrid_clock.cc:648] HybridClock initialized: now 1781945762799886 us; error 40 us; skew 500 ppm
08:56:02.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:02 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:02.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.802781 1463 init.cc:377] Logged in from keytab as kudu/127.1.8.60@KRBTEST.COM (short username kudu)
08:56:02.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.803772 1463 webserver.cc:460] Webserver started at http://127.1.8.60:38625/ using document root <none> and password file <none>
08:56:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.804345 1463 fs_manager.cc:359] Metadata directory not provided
08:56:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.804452 1463 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:56:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.808166 1463 fs_manager.cc:709] Time spent opening directory manager: real 0.002s user 0.004s sys 0.000s
08:56:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.810320 1478 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:56:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.811578 1463 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.811759 1463 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data,/tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "366e50900d05499fbee1850e1cd07db6"
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 08:56:02 on dist-test-slave-5mp7"
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.812328 1463 fs_report.cc:383] FS layout report
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/wal
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/data
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:56:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:56:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:56:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:56:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.828301 1463 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
08:56:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.831934 1463 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:56:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.832134 1463 kserver.cc:163] Server-wide thread pool size limit: 3276
08:56:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.839752 1463 rpc_server.cc:307] RPC server started. Bound to: 127.1.8.60:41503
08:56:02.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.839994 1530 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.8.60:41503 every 8 connection(s)
08:56:02.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.840924 1463 server_base.cc:1166] Dumped server information to /tmp/dist-test-task5KApzg/test-tmp/mini-kudu-cluster5992065680082027145/master-2/data/info.pb
08:56:02.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.843029 1056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task5KApzg/build/debug/bin/kudu as pid 1463
08:56:02.843 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:56:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.845973 1531 sys_catalog.cc:263] Verifying existing consensus state
08:56:02.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.848415 1531 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: Bootstrap starting.
08:56:02.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.865353 1531 log.cc:824] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: Log is configured to *not* fsync() on all Append() calls
08:56:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.867249 1531 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:56:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.867724 1531 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: Bootstrap complete.
08:56:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.873881 1531 raft_consensus.cc:348] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.874650 1531 raft_consensus.cc:374] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:56:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.874869 1531 raft_consensus.cc:729] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 366e50900d05499fbee1850e1cd07db6, State: Initialized, Role: FOLLOWER
08:56:02.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.875761 1531 consensus_queue.cc:260] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:02.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.876489 1535 sys_catalog.cc:455] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } } }
08:56:02.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.876634 1535 sys_catalog.cc:458] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [sys.catalog]: This master's current role is: FOLLOWER
08:56:02.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.878103 1531 sys_catalog.cc:564] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [sys.catalog]: configured and running, proceeding with master startup.
08:56:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.883411 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.883512 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.889523 1463 master_runner.cc:186] Error getting master registration for 127.1.8.62:41091: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111),
08:56:02.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.893216 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:56:02.900 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.919919 1535 raft_consensus.cc:482] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:56:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.920151 1535 raft_consensus.cc:504] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:02.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.921831 1535 leader_election.cc:288] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091), 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581)
08:56:02.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.922889 1480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111)
08:56:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.924269 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091): Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111)
08:56:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:02.924427 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581): Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.924476 1480 leader_election.cc:302] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 366e50900d05499fbee1850e1cd07db6; no voters: 14fe1efbcfb94507b4b151ed7c56e1b4, c35c491c2cbb47f29eed656fbf5ccbce
08:56:02.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:02.924752 1535 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:56:02.956 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:02.960 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.8.60:41503(127.1.8.60:41503)] connecting to peer
08:56:02.994 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:02.995 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:02.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.002 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.002 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.8.60:41503(127.1.8.60:41503)] Successfully connected to peer
08:56:03.070 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:56:03.078 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.8.60
08:56:03.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:03.083550 1484 tablet.cc:2369] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:56:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:03 dist-test-slave-5mp7 krb5kdc[1067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945762, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.8.60@KRBTEST.COM
08:56:03.272 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe30f72d9, L:/127.0.0.1:34362 - R:/127.1.8.60:41503] using SASL/GSSAPI
08:56:03.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.290 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:03.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:03.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.295 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.300 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.309 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:03.310 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:03.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.312 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.314 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.314 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.314 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.316 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.329 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:03.330 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:03.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.334 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.334 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.334 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.336 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.349 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:03.349 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:03.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.355 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.369 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:03.370 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:03.372 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.372 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.375 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.376 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.376 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.429 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:03.430 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:03.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.434 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.434 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.434 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.436 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:03.530 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:03.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.534 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.535 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.535 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.537 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:03.690 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:03.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:03.694 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.695 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.695 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:03.696 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:03.884756 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:03.884872 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:04.128 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:04.130 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:04.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.134 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.134 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.136 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:04.389 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:04.390 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:04.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.393 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:04.758826 1554 raft_consensus.cc:482] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:56:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:04.758960 1554 raft_consensus.cc:504] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:04.759409 1554 leader_election.cc:288] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091), 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581)
08:56:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:04.760980 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581): Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:04.761111 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091): Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111)
08:56:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:04.761154 1480 leader_election.cc:302] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 366e50900d05499fbee1850e1cd07db6; no voters: 14fe1efbcfb94507b4b151ed7c56e1b4, c35c491c2cbb47f29eed656fbf5ccbce
08:56:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:04.761307 1554 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:56:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:04.885650 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:04.885742 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:04.948 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:04.950 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:04.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:04.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:04.954 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:05.886509 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:05.886608 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:06.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:06.547430 1558 raft_consensus.cc:482] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:56:06.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:06.547571 1558 raft_consensus.cc:504] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:06.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:06.548128 1558 leader_election.cc:288] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091), 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581)
08:56:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:06.549611 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091): Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111)
08:56:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:06.549749 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581): Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:06.549796 1480 leader_election.cc:302] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 366e50900d05499fbee1850e1cd07db6; no voters: 14fe1efbcfb94507b4b151ed7c56e1b4, c35c491c2cbb47f29eed656fbf5ccbce
08:56:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:06.549934 1558 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:56:06.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:06.887303 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:06.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:06.887398 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:07.888257 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:07.888357 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:07.894479 1463 master_runner.cc:186] Error getting master registration for 127.1.8.62:41091: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111),
08:56:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:07.894613 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:56:08.649 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:08.650 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:08.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:08.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:08.652 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:08.653 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:08.653 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:08.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:08.654 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:08.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:08.716956 1563 raft_consensus.cc:482] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:56:08.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:08.717142 1563 raft_consensus.cc:504] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:08.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:08.717618 1563 leader_election.cc:288] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091), 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581)
08:56:08.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:08.718287 1480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111) [suppressed 5 similar messages]
08:56:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:08.719157 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091): Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111)
08:56:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:08.719275 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581): Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:08.719309 1480 leader_election.cc:302] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 366e50900d05499fbee1850e1cd07db6; no voters: 14fe1efbcfb94507b4b151ed7c56e1b4, c35c491c2cbb47f29eed656fbf5ccbce
08:56:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:08.719513 1563 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:56:08.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:08.889112 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:08.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:08.889212 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:09.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:09.889945 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:09.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:09.890048 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:10.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:10.890769 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:10.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:10.890868 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:11.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:11.891722 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:11.891819 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:12.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:12.550 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:12.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:12.551 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:12.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:12.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:12.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:12.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:12.554 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:12 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945772, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:12.804323 1474 init.cc:295] Successfully reacquired a new kerberos TGT
08:56:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:12.892565 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:12.892669 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:12.895619 1463 master_runner.cc:186] Error getting master registration for 127.1.8.62:41091: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111),
08:56:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:12.895714 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:56:13.489 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:13.490 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:13.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:13.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:13.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:13.493 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:13.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:13.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:13.494 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:13.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:13.893389 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:13.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:13.893491 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:14.894440 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:14.894563 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:15.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:15.772650 1567 raft_consensus.cc:482] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:56:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:15.772787 1567 raft_consensus.cc:504] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:15.773209 1567 leader_election.cc:288] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091), 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581)
08:56:15.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:15.774063 1480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111) [suppressed 1 similar messages]
08:56:15.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:15.774782 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091): Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111)
08:56:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:15.774902 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581): Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:15.774948 1480 leader_election.cc:302] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 366e50900d05499fbee1850e1cd07db6; no voters: 14fe1efbcfb94507b4b151ed7c56e1b4, c35c491c2cbb47f29eed656fbf5ccbce
08:56:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:15.775120 1567 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:56:15.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:15.895370 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:15.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:15.895467 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:16.896200 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:16.896294 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:17.009 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:17.010 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:17.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:17.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:17.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:17.013 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:17.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:17.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:17.014 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:17.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:17.530 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:17.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:17.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:17.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:17.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:17.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:17.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:17.534 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:17.897010 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:17.897111 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:17.901103 1463 master_runner.cc:186] Error getting master registration for 127.1.8.62:41091: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111),
08:56:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:17.901221 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:56:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:18.897871 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:18.897974 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:18.949 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:18.951 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:18.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:18.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:18.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:18.953 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:18.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:18.954 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:18.954 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:19.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:19.571062 1575 raft_consensus.cc:482] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:56:19.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:19.571198 1575 raft_consensus.cc:504] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:19.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:19.571707 1575 leader_election.cc:288] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091), 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581)
08:56:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:19.572857 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091): Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111)
08:56:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:19.572988 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581): Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:19.573024 1480 leader_election.cc:302] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 366e50900d05499fbee1850e1cd07db6; no voters: 14fe1efbcfb94507b4b151ed7c56e1b4, c35c491c2cbb47f29eed656fbf5ccbce
08:56:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:19.573169 1575 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:56:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:19.898798 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:19.898903 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:20.899706 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:20.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:20.899801 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:21.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:21.900538 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:21.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:21.900638 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:22.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:22.530 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:22.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:22.532 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:22.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:22.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:22.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:22.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:22.534 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:22.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 08:56:22 dist-test-slave-5mp7 krb5kdc[1067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781945782, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.8.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:56:22.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:22.805562 1474 init.cc:295] Successfully reacquired a new kerberos TGT
08:56:22.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:22.901386 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:22.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:22.901499 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:22.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:22.902173 1463 master_runner.cc:186] Error getting master registration for 127.1.8.62:41091: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111),
08:56:22.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:22.902290 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:56:23.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:23.902237 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:23.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:23.902336 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:24.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:24.089463 1580 raft_consensus.cc:482] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:56:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:24.089658 1580 raft_consensus.cc:504] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c35c491c2cbb47f29eed656fbf5ccbce" member_type: VOTER last_known_addr { host: "127.1.8.62" port: 41091 } } peers { permanent_uuid: "14fe1efbcfb94507b4b151ed7c56e1b4" member_type: VOTER last_known_addr { host: "127.1.8.61" port: 40581 } } peers { permanent_uuid: "366e50900d05499fbee1850e1cd07db6" member_type: VOTER last_known_addr { host: "127.1.8.60" port: 41503 } }
08:56:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:24.090111 1580 leader_election.cc:288] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091), 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581)
08:56:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:24.090854 1480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:56:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:24.091727 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c35c491c2cbb47f29eed656fbf5ccbce (127.1.8.62:41091): Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111)
08:56:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:24.091843 1480 leader_election.cc:334] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 14fe1efbcfb94507b4b151ed7c56e1b4 (127.1.8.61:40581): Network error: Client connection negotiation failed: client connection to 127.1.8.61:40581: connect: Connection refused (error 111)
08:56:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:24.091887 1480 leader_election.cc:302] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 366e50900d05499fbee1850e1cd07db6; no voters: 14fe1efbcfb94507b4b151ed7c56e1b4, c35c491c2cbb47f29eed656fbf5ccbce
08:56:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:24.092051 1580 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:56:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:24.903035 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:24.903131 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:25.889 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:25.890 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:25.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:25.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:25.891 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:25.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:25.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:25.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:25.894 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:25.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:25.903872 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:25.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:25.903972 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:26.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:26.904675 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:26.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:26.904778 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:27.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:27.903409 1463 master_runner.cc:186] Error getting master registration for 127.1.8.62:41091: Network error: Client connection negotiation failed: client connection to 127.1.8.62:41091: connect: Connection refused (error 111),
08:56:27.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:27.903532 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:56:27.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:27.905400 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:27.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:27.905467 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:28.906191 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:28.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:28.906291 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:29.149 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:29.150 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:29.150 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:29.151 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:29.151 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:29.151 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:29.151 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:29.152 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:29.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:29.907042 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:29.907145 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:30.907908 1546 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 366e50900d05499fbee1850e1cd07db6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:56:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 08:56:30.908006 1546 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:56:31.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.62:41091(127.1.8.62:41091)] connecting to peer
08:56:31.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.8.61:40581(127.1.8.61:40581)] connecting to peer
08:56:31.670 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:31.670 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.8.62:41091(127.1.8.62:41091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:31.671 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.62:41091: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091
08:56:31.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:31.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.61:40581(127.1.8.61:40581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:31.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.8.61:40581: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
08:56:31.674 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503; will retry
08:56:31.678 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [143ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [149ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [431ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [435ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [437ms] refreshing cache from master, [438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [439ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [441ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [448ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [449ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [450ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [456ms] refreshing cache from master, [457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [460ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [462ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [463ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [464ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [480ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [482ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [483ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [484ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [500ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [502ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [503ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [523ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [524ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [582ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [583ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [843ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [844ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [845ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1284ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1536ms] refreshing cache from master, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1539ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1541ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2096ms] refreshing cache from master, [2096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [2097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [2099ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [2100ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [2102ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [5796ms] refreshing cache from master, [5796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [5797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [5799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [5800ms] too many traces: truncated at 100 traces, deferred=Deferred@473418070(state=PENDING, result=null, callback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769), errback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769)))
08:56:31.683 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.8.62:41091(127.1.8.62:41091)] disconnecting while in state TERMINATED
08:56:31.684 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.8.61:40581(127.1.8.61:40581)] disconnecting while in state TERMINATED
08:56:31.684 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.8.60:41503(127.1.8.60:41503)] disconnecting while in state READY
08:56:31.689 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:56:31.691 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:56:31.692 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.8.60:41503(127.1.8.60:41503)] handling channelInactive
08:56:31.692 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.8.60:41503(127.1.8.60:41503)] cleaning up while in state READY due to: connection closed
08:56:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:31.692648 1056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 1463
08:56:31.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T08:56:31Z chronyd exiting
08:56:31.744 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:56:31.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:31.745568 367 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 650
08:56:31.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:31.751410 367 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 785
08:56:31.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:31.757951 367 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 920
08:56:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:31.763377 367 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 378
08:56:31.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:31.769925 367 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 447
08:56:31.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 08:56:31.775579 367 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task5KApzg/build/debug/bin/kudu with pid 519
08:56:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T08:56:31Z chronyd exiting
08:56:31.825 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [143ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [149ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [431ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [435ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [437ms] refreshing cache from master, [438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [439ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [441ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [448ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [449ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [450ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [456ms] refreshing cache from master, [457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [460ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [462ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [463ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [464ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [480ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [482ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [483ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [484ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [500ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [502ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [503ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [523ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [524ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [582ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [583ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [843ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [844ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [845ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1284ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1536ms] refreshing cache from master, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1539ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1541ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2096ms] refreshing cache from master, [2096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [2097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [2099ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [2100ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [2102ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [5796ms] refreshing cache from master, [5796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [5797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [5799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [5800ms] too many traces: truncated at 100 traces, deferred=Deferred@473418070(state=PENDING, result=null, callback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769), errback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.8.61:40581
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [143ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [149ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [431ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [435ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [437ms] refreshing cache from master, [438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [439ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [441ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [448ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [449ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [450ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [456ms] refreshing cache from master, [457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [460ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [462ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [463ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [464ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [480ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [482ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [483ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [484ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [500ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [502ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [503ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [523ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [524ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [582ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [583ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [843ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [844ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [845ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1284ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1536ms] refreshing cache from master, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1539ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1541ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2096ms] refreshing cache from master, [2096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [2097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [2099ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [2100ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [2102ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [5796ms] refreshing cache from master, [5796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [5797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [5799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [5800ms] too many traces: truncated at 100 traces, deferred=Deferred@473418070(state=PENDING, result=null, callback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769), errback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.8.61:40581
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
08:56:32.315 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
08:56:32.317 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [143ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [149ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [431ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [435ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [437ms] refreshing cache from master, [438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [439ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [441ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [448ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [449ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [450ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [456ms] refreshing cache from master, [457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [460ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [462ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [463ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [464ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [480ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [482ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [483ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [484ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [500ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [502ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [503ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [523ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [524ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [582ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [583ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [843ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [844ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [845ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1284ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1536ms] refreshing cache from master, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1539ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1541ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2096ms] refreshing cache from master, [2096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [2097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [2099ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [2100ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [2102ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [5796ms] refreshing cache from master, [5796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [5797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [5799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [5800ms] too many traces: truncated at 100 traces, deferred=Deferred@473418070(state=PENDING, result=null, callback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769), errback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.8.61:40581
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [143ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [149ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [431ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [435ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [437ms] refreshing cache from master, [438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [439ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [441ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [448ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [449ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [450ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [456ms] refreshing cache from master, [457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [460ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [462ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [463ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [464ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [480ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [482ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [483ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [484ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [500ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [502ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [503ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [523ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [524ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [582ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [583ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [836ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [839ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [843ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [844ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [845ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1282ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1284ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1536ms] refreshing cache from master, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [1538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [1539ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [1541ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2096ms] refreshing cache from master, [2096ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [2097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [2099ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.62:41091: Network error: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091, [2100ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.61:40581: Network error: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.8.60:41503: OK, [2102ms] delaying RPC due to: Service unavailable: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581, [5796ms] refreshing cache from master, [5796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.62:41091, [5797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.61:40581, [5799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.8.60:41503, [5800ms] too many traces: truncated at 100 traces, deferred=Deferred@473418070(state=PENDING, result=null, callback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769), errback=(continuation of Deferred@1454989675 after retry RPC after error@1193967590) -> (continuation of Deferred@1194059049 after retry RPC after error@1710549475) -> (continuation of Deferred@2027650575 after retry RPC after error@1813703613) -> (continuation of Deferred@1946175930 after retry RPC after error@922291496) -> (continuation of Deferred@359737086 after retry RPC after error@781021422) -> (continuation of Deferred@1488363804 after retry RPC after error@651720559) -> (continuation of Deferred@981660572 after retry RPC after error@2144202274) -> (continuation of Deferred@312348294 after retry RPC after error@2006474933) -> (continuation of Deferred@1512793033 after retry RPC after error@1241987664) -> (continuation of Deferred@672130984 after retry RPC after error@472295380) -> (continuation of Deferred@496155314 after retry RPC after error@1848952554) -> (continuation of Deferred@970018339 after retry RPC after error@283137738) -> (continuation of Deferred@414136654 after retry RPC after error@1357164327) -> (continuation of Deferred@1677153102 after retry RPC after error@1907564540) -> (continuation of Deferred@280440269 after retry RPC after error@1700738411) -> (continuation of Deferred@1013671853 after retry RPC after error@1431225123) -> (continuation of Deferred@2092844290 after retry RPC after error@1616298899) -> (continuation of Deferred@132613691 after retry RPC after error@605227583) -> (continuation of Deferred@713809386 after retry RPC after error@2090326387) -> (continuation of Deferred@1199197828 after retry RPC after error@2063918758) -> (continuation of Deferred@261361825 after retry RPC after error@1300892769)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.8.62:41091,127.1.8.61:40581,127.1.8.60:41503) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.62:41091(127.1.8.62:41091): Connection refused: /127.1.8.62:41091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.8.61:40581(127.1.8.61:40581): Connection refused: /127.1.8.61:40581
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.8.61:40581
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)