13:47:18.692 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
13:47:18.703 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskpcmmaJ/test-tmp
13:47:18.705 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:47:18.708 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskpcmmaJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:47:18.889 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
13:47:19.059 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:47:19.060 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:47:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.063838 23920 mini_oidc.cc:152] Starting JWKS server
13:47:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.066892 23920 webserver.cc:460] Webserver started at http://127.0.0.1:33181/ using document root <none> and password file <none>
13:47:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.067313 23920 mini_oidc.cc:189] Starting OIDC Discovery server
13:47:19.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.069309 23920 webserver.cc:460] Webserver started at http://127.0.0.1:37723/ using document root <none> and password file <none>
13:47:19.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:47:19Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:47:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:47:19Z Disabled control of system clock
13:47:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.123854 23920 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/wal
13:47:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/data
13:47:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/logs
13:47:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/data/info.pb
13:47:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.62:42945
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.62
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33181/jwks.json with env {}
13:47:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.387210 23932 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.387698 23932 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.388338 23932 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.414196 23932 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.414445 23932 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.414614 23932 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.414847 23932 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.415124 23932 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.415357 23932 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33181/jwks.json
13:47:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.444165 23932 master_runner.cc:386] Master server non-default flags:
13:47:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:19.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:19.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/data
13:47:19.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/wal
13:47:19.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:19.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:19.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:19.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.62:42945
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33181/jwks.json
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/data/info.pb
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.62
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/logs
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:19.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.445302 23932 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:19.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.447163 23932 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.463363 23941 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:20.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.463459 23938 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:20.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:19.463707 23939 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:20.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:19.464319 23932 server_base.cc:1034] running on GCE node
13:47:20.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.442490 23932 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.444849 23932 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:20.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.446149 23932 hybrid_clock.cc:648] HybridClock initialized: now 1714139240446125 us; error 40 us; skew 500 ppm
13:47:20.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.446928 23932 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.459302 23932 webserver.cc:460] Webserver started at http://127.23.92.62:45609/ using document root <none> and password file <none>
13:47:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.460060 23932 fs_manager.cc:359] Metadata directory not provided
13:47:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.460215 23932 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.460537 23932 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.464694 23932 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/data/instance:
13:47:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff60d7e213304beeb3fb4307fb2aab49"
13:47:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:20 on dist-test-slave-2jsl"
13:47:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.465713 23932 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/wal/instance:
13:47:20.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff60d7e213304beeb3fb4307fb2aab49"
13:47:20.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:20 on dist-test-slave-2jsl"
13:47:20.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.471820 23932 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
13:47:20.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.476691 23948 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:20.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.477625 23932 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
13:47:20.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.477941 23932 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/wal
13:47:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff60d7e213304beeb3fb4307fb2aab49"
13:47:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:20 on dist-test-slave-2jsl"
13:47:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.478338 23932 fs_report.cc:383] FS layout report
13:47:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/wal
13:47:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/wal
13:47:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/data/data
13:47:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.522522 23932 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.523849 23932 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.524317 23932 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.587891 23932 rpc_server.cc:307] RPC server started. Bound to: 127.23.92.62:42945
13:47:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.587961 23999 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.92.62:42945 every 8 connection(s)
13:47:20.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.590548 23932 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/data/info.pb
13:47:20.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.596004 24000 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:47:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.601181 23920 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 23932
13:47:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.601567 23920 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-0/wal/instance
13:47:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.604789 23920 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/wal
13:47:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/data
13:47:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/logs
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/data/info.pb
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.61:43597
13:47:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.61
13:47:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33181/jwks.json with env {}
13:47:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.606230 24000 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } has no permanent_uuid. Determining permanent_uuid...
13:47:20.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.631865 24000 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } has no permanent_uuid. Determining permanent_uuid...
13:47:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.634359 23952 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.92.61:43597: connect: Connection refused (error 111)
13:47:20.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.638307 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.61:43597: Network error: Client connection negotiation failed: client connection to 127.23.92.61:43597: connect: Connection refused (error 111)
13:47:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.687713 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } attempt: 1
13:47:20.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.692379 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.61:43597: Network error: Client connection negotiation failed: client connection to 127.23.92.61:43597: connect: Connection refused (error 111)
13:47:20.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.760699 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } attempt: 2
13:47:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.764246 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.61:43597: Network error: Client connection negotiation failed: client connection to 127.23.92.61:43597: connect: Connection refused (error 111)
13:47:20.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.855582 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } attempt: 3
13:47:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.859961 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.61:43597: Network error: Client connection negotiation failed: client connection to 127.23.92.61:43597: connect: Connection refused (error 111)
13:47:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.886195 24002 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.886601 24002 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:20.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.887163 24002 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.911819 24002 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.912055 24002 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.912220 24002 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.912391 24002 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.912549 24002 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.912675 24002 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33181/jwks.json
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.939693 24002 master_runner.cc:386] Master server non-default flags:
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/data
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/wal
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.61:43597
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33181/jwks.json
13:47:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/data/info.pb
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.61
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/logs
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.940785 24002 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:20.942529 24002 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.958067 24011 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:21.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.003707 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } attempt: 4
13:47:21.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:21.009532 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.61:43597: Network error: Client connection negotiation failed: client connection to 127.23.92.61:43597: connect: Connection refused (error 111)
13:47:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.309013 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } attempt: 5
13:47:21.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:21.316185 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.61:43597: Network error: Client connection negotiation failed: client connection to 127.23.92.61:43597: connect: Connection refused (error 111)
13:47:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.863659 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } attempt: 6
13:47:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:21.874732 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.61:43597: Network error: Client connection negotiation failed: client connection to 127.23.92.61:43597: connect: Connection refused (error 111)
13:47:21.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.959389 24014 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:21.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:20.958827 24012 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:21.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:21.978206 24013 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1019 milliseconds
13:47:21.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.978303 24002 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:47:21.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.979519 24002 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:21.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.982290 24002 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:21.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.983708 24002 hybrid_clock.cc:648] HybridClock initialized: now 1714139241983641 us; error 63 us; skew 500 ppm
13:47:21.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.984416 24002 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:21.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.990839 24002 webserver.cc:460] Webserver started at http://127.23.92.61:43677/ using document root <none> and password file <none>
13:47:21.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.991580 24002 fs_manager.cc:359] Metadata directory not provided
13:47:21.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.991739 24002 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:21.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.992065 24002 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.996419 24002 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/data/instance:
13:47:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2efdee4b1e444f793d2cde38d091412"
13:47:21.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:21 on dist-test-slave-2jsl"
13:47:21.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:21.997470 24002 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/wal/instance:
13:47:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2efdee4b1e444f793d2cde38d091412"
13:47:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:21 on dist-test-slave-2jsl"
13:47:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.003577 24002 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
13:47:22.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.008931 24023 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.009871 24002 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
13:47:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.010175 24002 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/wal
13:47:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2efdee4b1e444f793d2cde38d091412"
13:47:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:21 on dist-test-slave-2jsl"
13:47:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.010442 24002 fs_report.cc:383] FS layout report
13:47:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/wal
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/wal
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/data/data
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:22.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.054493 24002 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:22.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.055698 24002 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:22.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.056113 24002 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:22.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.128772 24002 rpc_server.cc:307] RPC server started. Bound to: 127.23.92.61:43597
13:47:22.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.128839 24074 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.92.61:43597 every 8 connection(s)
13:47:22.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.132663 24002 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/data/info.pb
13:47:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.137645 23920 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24002
13:47:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.138005 23920 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-1/wal/instance
13:47:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.139492 24075 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.140030 23920 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/wal
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/data
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/logs
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/data/info.pb
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.60:40179
13:47:22.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.60
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33181/jwks.json with env {}
13:47:22.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.150578 24075 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } has no permanent_uuid. Determining permanent_uuid...
13:47:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.172637 24075 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } has no permanent_uuid. Determining permanent_uuid...
13:47:22.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.184711 24075 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } has no permanent_uuid. Determining permanent_uuid...
13:47:22.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.188311 24027 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:22.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.197947 24075 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.247452 24075 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 1
13:47:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.250769 24075 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.319159 24075 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 2
13:47:22.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.322922 24075 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:22.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.414311 24075 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 3
13:47:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.417685 24075 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.430244 24077 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.430691 24077 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.431203 24077 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.456943 24077 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.457186 24077 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.457417 24077 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.457607 24077 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.457808 24077 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.457953 24077 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33181/jwks.json
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.485666 24077 master_runner.cc:386] Master server non-default flags:
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/data
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/wal
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.60:40179
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33181/jwks.json
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/data/info.pb
13:47:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.60
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/logs
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.486668 24077 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.488338 24077 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:22.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.504943 24088 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:22.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.561122 24075 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 4
13:47:22.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.567320 24075 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:22.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.866842 24075 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 5
13:47:22.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.873085 24075 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:22.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.935221 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } attempt: 7
13:47:22.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:22.960080 24000 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } has no permanent_uuid. Determining permanent_uuid...
13:47:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.969013 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.027515 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 1
13:47:23.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.033419 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.114881 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 2
13:47:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.121032 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.206543 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 3
13:47:23.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.213104 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:23.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.353576 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 4
13:47:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.358598 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.420634 24075 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 6
13:47:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.428684 24075 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.504937 24087 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:22.509092 24090 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.553515 24089 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1047 milliseconds
13:47:23.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.553617 24077 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:47:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.554819 24077 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:23.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.557680 24077 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:23.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.559093 24077 hybrid_clock.cc:648] HybridClock initialized: now 1714139243559053 us; error 56 us; skew 500 ppm
13:47:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.559864 24077 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:23.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.566221 24077 webserver.cc:460] Webserver started at http://127.23.92.60:37579/ using document root <none> and password file <none>
13:47:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.567068 24077 fs_manager.cc:359] Metadata directory not provided
13:47:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.567243 24077 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.567636 24077 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:23.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.574553 24077 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/data/instance:
13:47:23.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48b1a504d52447caab48f173df7473f"
13:47:23.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:23 on dist-test-slave-2jsl"
13:47:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.575927 24077 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/wal/instance:
13:47:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48b1a504d52447caab48f173df7473f"
13:47:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:23 on dist-test-slave-2jsl"
13:47:23.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.584151 24077 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
13:47:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.590210 24102 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.592136 24077 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
13:47:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.592397 24077 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/wal
13:47:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c48b1a504d52447caab48f173df7473f"
13:47:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:23 on dist-test-slave-2jsl"
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.592710 24077 fs_report.cc:383] FS layout report
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/wal
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/wal
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/data/data
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:23.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.641952 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 5
13:47:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.646188 24000 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.23.92.60:40179: Network error: Client connection negotiation failed: client connection to 127.23.92.60:40179: connect: Connection refused (error 111)
13:47:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.658987 24077 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:23.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.660692 24077 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.661048 24077 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.732074 24077 rpc_server.cc:307] RPC server started. Bound to: 127.23.92.60:40179
13:47:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.732174 24153 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.92.60:40179 every 8 connection(s)
13:47:23.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.734469 24077 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/data/info.pb
13:47:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.740394 24154 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:47:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.740530 23920 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24077
13:47:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.740947 23920 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/master-2/wal/instance
13:47:23.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.755023 24154 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } has no permanent_uuid. Determining permanent_uuid...
13:47:23.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.781124 24154 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } has no permanent_uuid. Determining permanent_uuid...
13:47:23.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.789712 24154 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } has no permanent_uuid. Determining permanent_uuid...
13:47:23.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.803121 24154 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f: Bootstrap starting.
13:47:23.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.808395 24154 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f: Neither blocks nor log segments found. Creating new log.
13:47:23.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.809922 24154 log.cc:824] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f: Log is configured to *not* fsync() on all Append() calls
13:47:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.814705 24154 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f: No bootstrap required, opened a new log
13:47:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.833914 24154 raft_consensus.cc:348] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.834548 24154 raft_consensus.cc:374] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:47:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.834791 24154 raft_consensus.cc:729] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c48b1a504d52447caab48f173df7473f, State: Initialized, Role: FOLLOWER
13:47:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.835441 24154 consensus_queue.cc:260] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.837741 24163 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.838351 24163 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: This master's current role is: FOLLOWER
13:47:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.839032 24154 sys_catalog.cc:564] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: configured and running, proceeding with master startup.
13:47:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.863096 24174 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.863619 24174 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:23.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.859561 24077 master_runner.cc:186] Error getting master registration for 127.23.92.62:42945: OK, instance_id { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" instance_seqno: 1714139240569814 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:47:23.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:23.880539 24077 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:23.915736 24108 tablet.cc:2369] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:47:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.198494 24000 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 6
13:47:24.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.215878 24000 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: Bootstrap starting.
13:47:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.222945 24000 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: Neither blocks nor log segments found. Creating new log.
13:47:24.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.225538 24000 log.cc:824] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: Log is configured to *not* fsync() on all Append() calls
13:47:24.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.229862 24000 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: No bootstrap required, opened a new log
13:47:24.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.247817 24163 raft_consensus.cc:482] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:24.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.248332 24163 raft_consensus.cc:504] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.250424 24163 leader_election.cc:288] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff60d7e213304beeb3fb4307fb2aab49 (127.23.92.62:42945), b2efdee4b1e444f793d2cde38d091412 (127.23.92.61:43597)
13:47:24.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.258786 24000 raft_consensus.cc:348] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:24.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.259490 24000 raft_consensus.cc:374] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:47:24.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.259734 24000 raft_consensus.cc:729] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff60d7e213304beeb3fb4307fb2aab49, State: Initialized, Role: FOLLOWER
13:47:24.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.260502 24000 consensus_queue.cc:260] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:24.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.263351 24178 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:24.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.264708 24178 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [sys.catalog]: This master's current role is: FOLLOWER
13:47:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.265642 24000 sys_catalog.cc:564] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [sys.catalog]: configured and running, proceeding with master startup.
13:47:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.265805 23975 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c48b1a504d52447caab48f173df7473f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ff60d7e213304beeb3fb4307fb2aab49" is_pre_election: true
13:47:24.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.266582 23975 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c48b1a504d52447caab48f173df7473f in term 0.
13:47:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.267752 24104 leader_election.cc:302] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c48b1a504d52447caab48f173df7473f, ff60d7e213304beeb3fb4307fb2aab49; no voters:
13:47:24.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.268402 24163 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 0 FOLLOWER]: Leader pre-election won for term 1
13:47:24.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.275041 24163 raft_consensus.cc:482] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:47:24.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.275357 24163 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 0 FOLLOWER]: Advancing to term 1
13:47:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:24.282097 23954 tablet.cc:2369] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:47:24.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.283535 24163 raft_consensus.cc:504] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.296164 23975 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c48b1a504d52447caab48f173df7473f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ff60d7e213304beeb3fb4307fb2aab49"
13:47:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.296550 24163 leader_election.cc:288] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [CANDIDATE]: Term 1 election: Requested vote from peers ff60d7e213304beeb3fb4307fb2aab49 (127.23.92.62:42945), b2efdee4b1e444f793d2cde38d091412 (127.23.92.61:43597)
13:47:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.296666 23975 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [term 0 FOLLOWER]: Advancing to term 1
13:47:24.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.304383 23975 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c48b1a504d52447caab48f173df7473f in term 1.
13:47:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.305512 24104 leader_election.cc:302] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c48b1a504d52447caab48f173df7473f, ff60d7e213304beeb3fb4307fb2aab49; no voters:
13:47:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.306233 24163 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 1 FOLLOWER]: Leader election won for term 1
13:47:24.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.310357 24163 raft_consensus.cc:686] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [term 1 LEADER]: Becoming Leader. State: Replica: c48b1a504d52447caab48f173df7473f, State: Running, Role: LEADER
13:47:24.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.312805 24163 consensus_queue.cc:237] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.323215 24190 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: SysCatalogTable state changed. Reason: New leader c48b1a504d52447caab48f173df7473f. Latest consensus state: current_term: 1 leader_uuid: "c48b1a504d52447caab48f173df7473f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.324257 24190 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: This master's current role is: LEADER
13:47:24.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.334925 24193 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:47:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:24.339771 24189 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:24.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:24.340049 24189 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:24.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.341256 24193 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:47:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.359614 23975 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [term 1 FOLLOWER]: Refusing update from remote peer c48b1a504d52447caab48f173df7473f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:47:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.362888 24190 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:47:24.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.317103 23932 master_runner.cc:186] Error getting master registration for 127.23.92.61:43597: OK, instance_id { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" instance_seqno: 1714139242108005 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:47:24.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.378810 23932 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:24.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.400274 24178 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c48b1a504d52447caab48f173df7473f. Latest consensus state: current_term: 1 leader_uuid: "c48b1a504d52447caab48f173df7473f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:24.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.400897 24178 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [sys.catalog]: This master's current role is: FOLLOWER
13:47:24.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.405459 24194 mvcc.cc:205] Tried to move back new op lower bound from 7021114344876429312 to 7021114344715239424. Current Snapshot: MvccSnapshot[applied={T|T < 7021114344876429312}]
13:47:24.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.406534 24190 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c48b1a504d52447caab48f173df7473f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:24.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.407214 24190 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: This master's current role is: LEADER
13:47:24.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.411659 24193 catalog_manager.cc:1347] Generated new cluster ID: aa98df49962641d1a0e71a0a6d0b39ce
13:47:24.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.411898 24193 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:47:24.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.435827 24178 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c48b1a504d52447caab48f173df7473f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.436519 24178 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49 [sys.catalog]: This master's current role is: FOLLOWER
13:47:24.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.450701 24193 catalog_manager.cc:1370] Generated new certificate authority record
13:47:24.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.452332 24193 catalog_manager.cc:1504] Loading token signing keys...
13:47:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.475925 24193 catalog_manager.cc:5857] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f: Generated new TSK 0
13:47:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.476838 24193 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:47:24.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.489120 24075 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } attempt: 7
13:47:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.509958 24075 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412: Bootstrap starting.
13:47:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.518851 24075 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412: Neither blocks nor log segments found. Creating new log.
13:47:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.519505 23920 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/wal
13:47:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/data
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/logs
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/data/info.pb
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.1:0
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.92.1
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.1
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:24.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:47:24.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.520881 24075 log.cc:824] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412: Log is configured to *not* fsync() on all Append() calls
13:47:24.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.536608 24075 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412: No bootstrap required, opened a new log
13:47:24.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.568692 24075 raft_consensus.cc:348] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:24.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.569792 24075 raft_consensus.cc:374] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:47:24.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.571182 24075 raft_consensus.cc:729] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b2efdee4b1e444f793d2cde38d091412, State: Initialized, Role: FOLLOWER
13:47:24.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.572381 24075 consensus_queue.cc:260] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } }
13:47:24.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.586776 24202 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.588577 24202 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [sys.catalog]: This master's current role is: FOLLOWER
13:47:24.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.600363 24075 sys_catalog.cc:564] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [sys.catalog]: configured and running, proceeding with master startup.
13:47:24.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.600881 24048 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [term 0 FOLLOWER]: Advancing to term 1
13:47:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.602927 24050 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c48b1a504d52447caab48f173df7473f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b2efdee4b1e444f793d2cde38d091412" is_pre_election: true
13:47:24.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.604315 24049 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c48b1a504d52447caab48f173df7473f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b2efdee4b1e444f793d2cde38d091412"
13:47:24.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.610136 24048 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [term 1 FOLLOWER]: Refusing update from remote peer c48b1a504d52447caab48f173df7473f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:47:24.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.611704 24163 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [LEADER]: Connected to new peer: Peer: permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:47:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.680187 24202 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c48b1a504d52447caab48f173df7473f. Latest consensus state: current_term: 1 leader_uuid: "c48b1a504d52447caab48f173df7473f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:24.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.681243 24202 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [sys.catalog]: This master's current role is: FOLLOWER
13:47:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.682581 24208 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c48b1a504d52447caab48f173df7473f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.683485 24208 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412 [sys.catalog]: This master's current role is: FOLLOWER
13:47:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.815876 24218 catalog_manager.cc:1259] Loaded cluster ID: aa98df49962641d1a0e71a0a6d0b39ce
13:47:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.816463 24218 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412: loading cluster ID for follower catalog manager: success
13:47:24.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.822887 24218 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412: acquiring CA information for follower catalog manager: success
13:47:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:24.843461 24218 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b2efdee4b1e444f793d2cde38d091412: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:47:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:25.030421 24201 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:25.031097 24201 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:25.031795 24201 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:25.079795 24201 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:25.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:25.080909 24201 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.92.1
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.133733 24201 tablet_server_runner.cc:78] Tablet server non-default flags:
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/data
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/wal
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:25.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.1:0
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/data/info.pb
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.1
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.92.1
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/logs
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:47:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.135140 24201 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.137033 24201 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:25.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.168061 24163 sys_catalog.cc:455] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c48b1a504d52447caab48f173df7473f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff60d7e213304beeb3fb4307fb2aab49" member_type: VOTER last_known_addr { host: "127.23.92.62" port: 42945 } } peers { permanent_uuid: "b2efdee4b1e444f793d2cde38d091412" member_type: VOTER last_known_addr { host: "127.23.92.61" port: 43597 } } peers { permanent_uuid: "c48b1a504d52447caab48f173df7473f" member_type: VOTER last_known_addr { host: "127.23.92.60" port: 40179 } } }
13:47:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.169533 24163 sys_catalog.cc:458] T 00000000000000000000000000000000 P c48b1a504d52447caab48f173df7473f [sys.catalog]: This master's current role is: LEADER
13:47:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:25.191689 24226 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.345490 24189 catalog_manager.cc:1259] Loaded cluster ID: aa98df49962641d1a0e71a0a6d0b39ce
13:47:25.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.346097 24189 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: loading cluster ID for follower catalog manager: success
13:47:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.353365 24189 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: acquiring CA information for follower catalog manager: success
13:47:25.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:25.370558 24189 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ff60d7e213304beeb3fb4307fb2aab49: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:47:27.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:25.194352 24228 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:27.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:26.572892 24225 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 24201
13:47:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.076707 24225 kernel_stack_watchdog.cc:198] Thread 24201 stuck at /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:637 for 384ms:
13:47:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
13:47:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
13:47:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
13:47:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
13:47:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.077567 24201 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.891s user 0.355s sys 0.571s
13:47:27.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.077903 24201 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.891s user 0.355s sys 0.571s
13:47:27.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.085752 24201 server_base.cc:1034] running on GCE node
13:47:27.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.086266 24232 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:27.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.087260 24201 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:27.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.089627 24201 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.091030 24201 hybrid_clock.cc:648] HybridClock initialized: now 1714139247090921 us; error 104 us; skew 500 ppm
13:47:27.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.092020 24201 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:27.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.104247 24201 webserver.cc:460] Webserver started at http://127.23.92.1:38841/ using document root <none> and password file <none>
13:47:27.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.105376 24201 fs_manager.cc:359] Metadata directory not provided
13:47:27.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.105612 24201 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:27.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.106079 24201 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.110442 24201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/data/instance:
13:47:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6aba0f318764e8e8bd6d865c10cbe6d"
13:47:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:27 on dist-test-slave-2jsl"
13:47:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.111737 24201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/wal/instance:
13:47:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6aba0f318764e8e8bd6d865c10cbe6d"
13:47:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:27 on dist-test-slave-2jsl"
13:47:27.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.119554 24201 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
13:47:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.125828 24237 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.126981 24201 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
13:47:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.127427 24201 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/wal
13:47:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6aba0f318764e8e8bd6d865c10cbe6d"
13:47:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:27 on dist-test-slave-2jsl"
13:47:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.128057 24201 fs_report.cc:383] FS layout report
13:47:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/wal
13:47:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/wal
13:47:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/data/data
13:47:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:27.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.191269 24201 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:27.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.193396 24201 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:27.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.193847 24201 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.196463 24201 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:47:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.201248 24201 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:47:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.201442 24201 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:47:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.201736 24201 ts_tablet_manager.cc:590] Registered 0 tablets
13:47:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.201885 24201 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:47:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.386049 24201 rpc_server.cc:307] RPC server started. Bound to: 127.23.92.1:46527
13:47:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.386198 24349 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.92.1:46527 every 8 connection(s)
13:47:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.388593 24201 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/data/info.pb
13:47:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.391753 23920 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24201
13:47:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.392179 23920 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-0/wal/instance
13:47:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.406399 23920 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/wal
13:47:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/data
13:47:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/logs
13:47:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/data/info.pb
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.2:0
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.92.2
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.2
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:47:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.437112 24350 heartbeater.cc:347] Connected to a master server at 127.23.92.60:40179
13:47:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.438189 24350 heartbeater.cc:464] Registering TS with master...
13:47:27.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.439885 24350 heartbeater.cc:511] Master 127.23.92.60:40179 requested a full tablet report, sending...
13:47:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.442333 24351 heartbeater.cc:347] Connected to a master server at 127.23.92.61:43597
13:47:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.442656 24351 heartbeater.cc:464] Registering TS with master...
13:47:27.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.443147 24119 ts_manager.cc:194] Registered new tserver with Master: b6aba0f318764e8e8bd6d865c10cbe6d (127.23.92.1:46527)
13:47:27.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.443935 24351 heartbeater.cc:511] Master 127.23.92.61:43597 requested a full tablet report, sending...
13:47:27.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.445564 24119 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.92.1:45973
13:47:27.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.446636 24040 ts_manager.cc:194] Registered new tserver with Master: b6aba0f318764e8e8bd6d865c10cbe6d (127.23.92.1:46527)
13:47:27.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.456009 24355 heartbeater.cc:347] Connected to a master server at 127.23.92.62:42945
13:47:27.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.456342 24355 heartbeater.cc:464] Registering TS with master...
13:47:27.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.456898 24355 heartbeater.cc:511] Master 127.23.92.62:42945 requested a full tablet report, sending...
13:47:27.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.459324 23965 ts_manager.cc:194] Registered new tserver with Master: b6aba0f318764e8e8bd6d865c10cbe6d (127.23.92.1:46527)
13:47:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.706629 24354 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.707008 24354 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.707376 24354 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:27.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.732718 24354 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.733496 24354 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.92.2
13:47:27.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.761615 24354 tablet_server_runner.cc:78] Tablet server non-default flags:
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/data
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/wal
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.2:0
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/data/info.pb
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.2
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.92.2
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/logs
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.762894 24354 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.764731 24354 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.781339 24366 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.448889 24350 heartbeater.cc:503] Master 127.23.92.60:40179 was elected leader, sending a full tablet report...
13:47:28.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.782275 24365 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:28.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:27.782326 24368 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:28.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:27.783474 24354 server_base.cc:1034] running on GCE node
13:47:28.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.766809 24354 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:28.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.769084 24354 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:28.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.770463 24354 hybrid_clock.cc:648] HybridClock initialized: now 1714139248770429 us; error 45 us; skew 500 ppm
13:47:28.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.771168 24354 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:28.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.777874 24354 webserver.cc:460] Webserver started at http://127.23.92.2:42237/ using document root <none> and password file <none>
13:47:28.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.779043 24354 fs_manager.cc:359] Metadata directory not provided
13:47:28.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.779242 24354 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:28.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.779651 24354 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.784216 24354 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/data/instance:
13:47:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98bc964b142b46ed98ca66b92a43fff6"
13:47:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:28 on dist-test-slave-2jsl"
13:47:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.785176 24354 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/wal/instance:
13:47:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98bc964b142b46ed98ca66b92a43fff6"
13:47:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:28 on dist-test-slave-2jsl"
13:47:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.791635 24354 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
13:47:28.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.797305 24375 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.798240 24354 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
13:47:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.798547 24354 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/wal
13:47:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98bc964b142b46ed98ca66b92a43fff6"
13:47:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:28 on dist-test-slave-2jsl"
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.798867 24354 fs_report.cc:383] FS layout report
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/wal
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/wal
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/data/data
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:28.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.860726 24354 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:28.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.862466 24354 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:28.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.862874 24354 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:28.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.864954 24354 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:47:28.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.869194 24354 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:47:28.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.869393 24354 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:47:28.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.869585 24354 ts_tablet_manager.cc:590] Registered 0 tablets
13:47:28.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:28.869715 24354 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:47:29.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.015506 24354 rpc_server.cc:307] RPC server started. Bound to: 127.23.92.2:39109
13:47:29.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.015591 24488 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.92.2:39109 every 8 connection(s)
13:47:29.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.017917 24354 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/data/info.pb
13:47:29.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.021481 23920 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24354
13:47:29.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.022070 23920 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-1/wal/instance
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.046375 23920 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/wal
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/data
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/logs
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/data/info.pb
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.3:0
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.92.3
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.3
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:47:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.055073 24490 heartbeater.cc:347] Connected to a master server at 127.23.92.61:43597
13:47:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.055483 24490 heartbeater.cc:464] Registering TS with master...
13:47:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.056658 24490 heartbeater.cc:511] Master 127.23.92.61:43597 requested a full tablet report, sending...
13:47:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.058619 24489 heartbeater.cc:347] Connected to a master server at 127.23.92.60:40179
13:47:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.059338 24489 heartbeater.cc:464] Registering TS with master...
13:47:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.060225 24040 ts_manager.cc:194] Registered new tserver with Master: 98bc964b142b46ed98ca66b92a43fff6 (127.23.92.2:39109)
13:47:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.060196 24489 heartbeater.cc:511] Master 127.23.92.60:40179 requested a full tablet report, sending...
13:47:29.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.062176 24119 ts_manager.cc:194] Registered new tserver with Master: 98bc964b142b46ed98ca66b92a43fff6 (127.23.92.2:39109)
13:47:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.063876 24119 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.92.2:44193
13:47:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.064744 24491 heartbeater.cc:347] Connected to a master server at 127.23.92.62:42945
13:47:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.065119 24491 heartbeater.cc:464] Registering TS with master...
13:47:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.066154 24491 heartbeater.cc:511] Master 127.23.92.62:42945 requested a full tablet report, sending...
13:47:29.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.068141 23965 ts_manager.cc:194] Registered new tserver with Master: 98bc964b142b46ed98ca66b92a43fff6 (127.23.92.2:39109)
13:47:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:29.364158 24499 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:29.364562 24499 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:29.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:29.364984 24499 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:29.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:29.400352 24499 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:29.401124 24499 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.92.3
13:47:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.432503 24499 tablet_server_runner.cc:78] Tablet server non-default flags:
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.92.20:41415
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/data
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/wal
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.92.3:0
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/data/info.pb
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.92.3
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.92.62:42945,127.23.92.61:43597,127.23.92.60:40179
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.92.3
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/logs
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.433519 24499 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.435173 24499 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:29.450996 24506 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:30.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.068714 24489 heartbeater.cc:503] Master 127.23.92.60:40179 was elected leader, sending a full tablet report...
13:47:30.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:29.451449 24509 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:30.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:29.458078 24499 server_base.cc:1034] running on GCE node
13:47:30.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:29.452471 24507 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.476336 24499 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.478614 24499 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.479988 24499 hybrid_clock.cc:648] HybridClock initialized: now 1714139250479956 us; error 33 us; skew 500 ppm
13:47:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.480760 24499 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:30.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.487301 24499 webserver.cc:460] Webserver started at http://127.23.92.3:44493/ using document root <none> and password file <none>
13:47:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.488312 24499 fs_manager.cc:359] Metadata directory not provided
13:47:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.488533 24499 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.488991 24499 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.493722 24499 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/data/instance:
13:47:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "061ed9e41f31440b81958c371def6ec0"
13:47:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:30 on dist-test-slave-2jsl"
13:47:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.494835 24499 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/wal/instance:
13:47:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "061ed9e41f31440b81958c371def6ec0"
13:47:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:30 on dist-test-slave-2jsl"
13:47:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.501371 24499 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.005s
13:47:30.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.506893 24516 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:30.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.507886 24499 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
13:47:30.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.508281 24499 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/wal
13:47:30.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "061ed9e41f31440b81958c371def6ec0"
13:47:30.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:30 on dist-test-slave-2jsl"
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.508622 24499 fs_report.cc:383] FS layout report
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/wal
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/wal
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/data/data
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.597505 24499 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.599014 24499 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.599467 24499 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.602823 24499 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:47:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.607417 24499 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:47:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.607676 24499 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:47:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.607977 24499 ts_tablet_manager.cc:590] Registered 0 tablets
13:47:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.608163 24499 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:47:30.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.759557 24499 rpc_server.cc:307] RPC server started. Bound to: 127.23.92.3:39113
13:47:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.759634 24628 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.92.3:39113 every 8 connection(s)
13:47:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.761757 24499 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/data/info.pb
13:47:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.768383 23920 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24499
13:47:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.768982 23920 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster5685409448353472949/ts-2/wal/instance
13:47:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.801892 24629 heartbeater.cc:347] Connected to a master server at 127.23.92.60:40179
13:47:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.802565 24629 heartbeater.cc:464] Registering TS with master...
13:47:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.804185 24629 heartbeater.cc:511] Master 127.23.92.60:40179 requested a full tablet report, sending...
13:47:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.806375 24631 heartbeater.cc:347] Connected to a master server at 127.23.92.62:42945
13:47:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.806669 24631 heartbeater.cc:464] Registering TS with master...
13:47:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.806927 24118 ts_manager.cc:194] Registered new tserver with Master: 061ed9e41f31440b81958c371def6ec0 (127.23.92.3:39113)
13:47:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.807456 24631 heartbeater.cc:511] Master 127.23.92.62:42945 requested a full tablet report, sending...
13:47:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.808076 24630 heartbeater.cc:347] Connected to a master server at 127.23.92.61:43597
13:47:30.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.808674 24118 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.92.3:48095
13:47:30.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.808446 24630 heartbeater.cc:464] Registering TS with master...
13:47:30.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.809623 24630 heartbeater.cc:511] Master 127.23.92.61:43597 requested a full tablet report, sending...
13:47:30.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.810101 23965 ts_manager.cc:194] Registered new tserver with Master: 061ed9e41f31440b81958c371def6ec0 (127.23.92.3:39113)
13:47:30.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.812551 24040 ts_manager.cc:194] Registered new tserver with Master: 061ed9e41f31440b81958c371def6ec0 (127.23.92.3:39113)
13:47:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:30.817399 23920 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
13:47:30.827 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:47:30.828 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
13:47:30.838 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.92.62"
port: 42945
}
bound_http_address {
host: "127.23.92.62"
port: 45609
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.92.61"
port: 43597
}
bound_http_address {
host: "127.23.92.61"
port: 43677
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.92.60"
port: 40179
}
bound_http_address {
host: "127.23.92.60"
port: 37579
}
}
}
13:47:30.843 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:47:30.845 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.92.1"
port: 46527
}
bound_http_address {
host: "127.23.92.1"
port: 38841
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.92.2"
port: 39109
}
bound_http_address {
host: "127.23.92.2"
port: 42237
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.92.3"
port: 39113
}
bound_http_address {
host: "127.23.92.3"
port: 44493
}
}
}
13:47:30.846 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
13:47:31.002 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
13:47:31.100 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskpcmmaJ/test-tmp
13:47:31.101 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:47:31.101 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskpcmmaJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:47:31.107 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
13:47:31.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:31.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:47:31.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:31.405059 24641 mini_oidc.cc:152] Starting JWKS server
13:47:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:31.408257 24641 webserver.cc:460] Webserver started at http://127.0.0.1:46665/ using document root <none> and password file <none>
13:47:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:31.408620 24641 mini_oidc.cc:189] Starting OIDC Discovery server
13:47:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:31.410761 24641 webserver.cc:460] Webserver started at http://127.0.0.1:46739/ using document root <none> and password file <none>
13:47:31.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
13:47:31.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/principal' for realm 'KRBTEST.COM',
13:47:31.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
13:47:31.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:31 dist-test-slave-2jsl krb5kdc[24653](info): setting up network...
13:47:31.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
13:47:31.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:31 dist-test-slave-2jsl krb5kdc[24653](info): set up 2 sockets
13:47:31.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:31 dist-test-slave-2jsl krb5kdc[24653](info): commencing operation
13:47:31.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
13:47:31.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:31.812126 24629 heartbeater.cc:503] Master 127.23.92.60:40179 was elected leader, sending a full tablet report...
13:47:32.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.383525 24641 mini_kdc.cc:121] Time spent starting KDC: real 0.937s user 0.005s sys 0.013s
13:47:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
13:47:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:47:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
13:47:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
13:47:32.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:47:32.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
13:47:32.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
13:47:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:47:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
13:47:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:47:32.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/test-user.keytab.
13:47:32.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/test-user.keytab.
13:47:32.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:32 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139252, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:32.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:47:32.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:47:32Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:47:32.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:47:32Z Disabled control of system clock
13:47:32.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.16.126@KRBTEST.COM; defaulting to no policy
13:47:32.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:47:32.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.16.126@KRBTEST.COM" created.
13:47:32.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:47:32.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.16.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.126.keytab.
13:47:32.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.16.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.126.keytab.
13:47:32.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:32.548557 24641 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:32.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:32.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/logs
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/info.pb
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.126:35785
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.126
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.126.keytab
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.126
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:47:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.817543 24667 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.818550 24667 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.819334 24667 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:32.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.846377 24667 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:47:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.846572 24667 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.846930 24667 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.847184 24667 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.847453 24667 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.847759 24667 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:32.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.847949 24667 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46665/jwks.json
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:32.877106 24667 master_runner.cc:386] Master server non-default flags:
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.126.keytab
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.126
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.126:35785
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/info.pb
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.126
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/logs
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:32.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:32.878396 24667 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:32.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:32.879503 24667 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.894014 24673 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.894488 24674 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:32.898093 24667 server_base.cc:1034] running on GCE node
13:47:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:32.894802 24676 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.929056 24667 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.932739 24667 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:33.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.934192 24667 hybrid_clock.cc:648] HybridClock initialized: now 1714139253934147 us; error 50 us; skew 500 ppm
13:47:33.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.934986 24667 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:33 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139253, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:33.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.940546 24667 init.cc:377] Logged in from keytab as kudu/127.24.16.126@KRBTEST.COM (short username kudu)
13:47:33.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.945832 24667 webserver.cc:460] Webserver started at http://127.24.16.126:34261/ using document root <none> and password file <none>
13:47:33.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.946758 24667 fs_manager.cc:359] Metadata directory not provided
13:47:33.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.946943 24667 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:33.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.947353 24667 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:33.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.952409 24667 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/instance:
13:47:33.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f316ad4a477145778a0330804f4c18f9"
13:47:33.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:33 on dist-test-slave-2jsl"
13:47:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.953320 24667 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal/instance:
13:47:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f316ad4a477145778a0330804f4c18f9"
13:47:33.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:33 on dist-test-slave-2jsl"
13:47:33.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.959826 24667 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
13:47:33.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.965142 24684 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.966045 24667 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
13:47:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.966380 24667 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f316ad4a477145778a0330804f4c18f9"
13:47:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:33 on dist-test-slave-2jsl"
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:33.966661 24667 fs_report.cc:383] FS layout report
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/data
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:33.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.013334 24667 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.018821 24667 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.019302 24667 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:34.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.096772 24667 rpc_server.cc:307] RPC server started. Bound to: 127.24.16.126:35785
13:47:34.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.096834 24735 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.16.126:35785 every 8 connection(s)
13:47:34.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.099077 24667 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/info.pb
13:47:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.104676 24736 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:47:34.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.106361 24641 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24667
13:47:34.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.106674 24641 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal/instance
13:47:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.115052 24736 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } has no permanent_uuid. Determining permanent_uuid...
13:47:34.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.16.125@KRBTEST.COM; defaulting to no policy
13:47:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:47:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.16.125@KRBTEST.COM" created.
13:47:34.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:34 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139253, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.126@KRBTEST.COM for kudu/127.24.16.126@KRBTEST.COM
13:47:34.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:47:34.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.16.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.125.keytab.
13:47:34.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.16.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.125.keytab.
13:47:34.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.150928 24736 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } has no permanent_uuid. Determining permanent_uuid...
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.152369 24641 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:34.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/logs
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/info.pb
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.125:34507
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.125
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.125.keytab
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.125
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:47:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.154291 24686 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.158087 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.125:34507: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.208106 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } attempt: 1
13:47:34.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.211941 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.125:34507: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.280323 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } attempt: 2
13:47:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.283995 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.125:34507: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.375315 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } attempt: 3
13:47:34.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.379206 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.125:34507: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:34.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.432801 24743 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:34.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.433228 24743 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:34.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.433749 24743 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.460327 24743 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:47:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.460496 24743 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.460721 24743 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.460886 24743 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.461047 24743 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.461202 24743 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.461321 24743 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46665/jwks.json
13:47:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.491588 24743 master_runner.cc:386] Master server non-default flags:
13:47:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.125.keytab
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.125
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.125:34507
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/info.pb
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.125
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/logs
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.492544 24743 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.493494 24743 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:34.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.508952 24752 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:34.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.522516 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } attempt: 4
13:47:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.525482 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.125:34507: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:34.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.824923 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } attempt: 5
13:47:34.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.829443 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.125:34507: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.377002 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } attempt: 6
13:47:35.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:35.383502 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.125:34507: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:35.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.509058 24749 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:35.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:34.510929 24743 server_base.cc:1034] running on GCE node
13:47:35.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:34.510875 24750 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.534310 24743 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.536712 24743 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:35.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.538053 24743 hybrid_clock.cc:648] HybridClock initialized: now 1714139255538014 us; error 44 us; skew 500 ppm
13:47:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.538823 24743 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:35 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139255, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:35.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.543963 24743 init.cc:377] Logged in from keytab as kudu/127.24.16.125@KRBTEST.COM (short username kudu)
13:47:35.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.549453 24743 webserver.cc:460] Webserver started at http://127.24.16.125:38657/ using document root <none> and password file <none>
13:47:35.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.550328 24743 fs_manager.cc:359] Metadata directory not provided
13:47:35.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.550493 24743 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:35.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.550839 24743 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.554931 24743 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/instance:
13:47:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f9c789a9f0e41d3adc7f0dbb5114073"
13:47:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:35 on dist-test-slave-2jsl"
13:47:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.556006 24743 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal/instance:
13:47:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f9c789a9f0e41d3adc7f0dbb5114073"
13:47:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:35 on dist-test-slave-2jsl"
13:47:35.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.562381 24743 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
13:47:35.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.566982 24763 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.568033 24743 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
13:47:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.568404 24743 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f9c789a9f0e41d3adc7f0dbb5114073"
13:47:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:35 on dist-test-slave-2jsl"
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.568719 24743 fs_report.cc:383] FS layout report
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/data
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.616142 24743 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:35.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.621621 24743 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.622052 24743 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:35.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.696168 24743 rpc_server.cc:307] RPC server started. Bound to: 127.24.16.125:34507
13:47:35.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.696274 24814 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.16.125:34507 every 8 connection(s)
13:47:35.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.698571 24743 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/info.pb
13:47:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.701517 24641 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24743
13:47:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.702018 24641 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal/instance
13:47:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.705202 24815 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:47:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.16.124@KRBTEST.COM; defaulting to no policy
13:47:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.716456 24815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } has no permanent_uuid. Determining permanent_uuid...
13:47:35.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:47:35.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.16.124@KRBTEST.COM" created.
13:47:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:35 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139255, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.125@KRBTEST.COM for kudu/127.24.16.126@KRBTEST.COM
13:47:35.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:47:35.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.16.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.124.keytab.
13:47:35.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.16.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.124.keytab.
13:47:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.745546 24641 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/logs
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/info.pb
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.124:43921
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.124
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.124.keytab
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.124
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:35.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:47:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.750113 24815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } has no permanent_uuid. Determining permanent_uuid...
13:47:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:35 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139255, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.125@KRBTEST.COM for kudu/127.24.16.125@KRBTEST.COM
13:47:35.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.776015 24815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } has no permanent_uuid. Determining permanent_uuid...
13:47:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:35.778254 24767 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:35.781908 24815 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:35.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.831336 24815 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 1
13:47:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:35.834405 24815 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:35.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.902681 24815 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 2
13:47:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:35.905886 24815 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:35.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:35.997232 24815 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 3
13:47:36.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.001288 24815 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.039731 24821 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:36.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.040288 24821 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:36.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.041071 24821 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:36.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.066661 24821 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:47:36.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.066872 24821 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:36.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.067214 24821 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:36.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.067467 24821 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:36.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.067716 24821 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:36.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.067963 24821 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:36.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.068145 24821 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46665/jwks.json
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.099126 24821 master_runner.cc:386] Master server non-default flags:
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.124.keytab
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.124
13:47:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.124:43921
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/info.pb
13:47:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.124
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/logs
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.100407 24821 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:36.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.101572 24821 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:36.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.120483 24832 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:36.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.144634 24815 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 4
13:47:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.149992 24815 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.444036 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } attempt: 7
13:47:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.449388 24815 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 5
13:47:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.454766 24815 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:36.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:36 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139253, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.126@KRBTEST.COM for kudu/127.24.16.125@KRBTEST.COM
13:47:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.473421 24736 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } has no permanent_uuid. Determining permanent_uuid...
13:47:36.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.479595 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.537927 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 1
13:47:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.542560 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.623940 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 2
13:47:36.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.628477 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.714097 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 3
13:47:36.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.719228 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.859581 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 4
13:47:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.863749 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:37.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.002841 24815 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 6
13:47:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.009037 24815 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:37.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.147087 24736 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } attempt: 5
13:47:37.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.151192 24736 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.24.16.124:43921: Network error: Client connection negotiation failed: client connection to 127.24.16.124:43921: connect: Connection refused (error 111)
13:47:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.120572 24830 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:36.121611 24821 server_base.cc:1034] running on GCE node
13:47:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:36.121052 24829 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:37.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.160598 24821 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:37.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.163575 24821 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:37.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.164974 24821 hybrid_clock.cc:648] HybridClock initialized: now 1714139257164931 us; error 45 us; skew 500 ppm
13:47:37.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.165632 24821 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:37.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139257, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.171784 24821 init.cc:377] Logged in from keytab as kudu/127.24.16.124@KRBTEST.COM (short username kudu)
13:47:37.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.176657 24821 webserver.cc:460] Webserver started at http://127.24.16.124:44159/ using document root <none> and password file <none>
13:47:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.177426 24821 fs_manager.cc:359] Metadata directory not provided
13:47:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.177587 24821 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:37.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.177927 24821 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:47:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.182348 24821 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/instance:
13:47:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "054d1028f8874ff39eecd59abc437469"
13:47:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:37 on dist-test-slave-2jsl"
13:47:37.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.183243 24821 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal/instance:
13:47:37.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "054d1028f8874ff39eecd59abc437469"
13:47:37.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:37 on dist-test-slave-2jsl"
13:47:37.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.189529 24821 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
13:47:37.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.194977 24845 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.195878 24821 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.196136 24821 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "054d1028f8874ff39eecd59abc437469"
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:37 on dist-test-slave-2jsl"
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.196377 24821 fs_report.cc:383] FS layout report
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/data
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:37.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:37.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:37.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:37.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.255656 24821 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.260116 24821 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:37.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.260437 24821 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:37.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.338443 24821 rpc_server.cc:307] RPC server started. Bound to: 127.24.16.124:43921
13:47:37.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.338516 24896 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.16.124:43921 every 8 connection(s)
13:47:37.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.341087 24821 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/info.pb
13:47:37.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.346391 24641 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24821
13:47:37.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.346829 24641 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal/instance
13:47:37.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.346889 24897 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:47:37.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.360383 24897 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } has no permanent_uuid. Determining permanent_uuid...
13:47:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139252, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.16.124@KRBTEST.COM
13:47:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139257, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.124@KRBTEST.COM for kudu/127.24.16.126@KRBTEST.COM
13:47:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.401130 24897 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } has no permanent_uuid. Determining permanent_uuid...
13:47:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139252, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.16.125@KRBTEST.COM
13:47:37.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139257, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.124@KRBTEST.COM for kudu/127.24.16.125@KRBTEST.COM
13:47:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.421897 24897 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } has no permanent_uuid. Determining permanent_uuid...
13:47:37.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139252, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.16.126@KRBTEST.COM
13:47:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139257, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.124@KRBTEST.COM for kudu/127.24.16.124@KRBTEST.COM
13:47:37.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.444576 24641 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
13:47:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.448125 24897 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Bootstrap starting.
13:47:37.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:37.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
13:47:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.454631 24897 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Neither blocks nor log segments found. Creating new log.
13:47:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.456902 24897 log.cc:824] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Log is configured to *not* fsync() on all Append() calls
13:47:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.462556 24897 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: No bootstrap required, opened a new log
13:47:37.466 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
13:47:37.467 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
13:47:37.471 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.16.126"
port: 35785
}
bound_http_address {
host: "127.24.16.126"
port: 34261
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.16.125"
port: 34507
}
bound_http_address {
host: "127.24.16.125"
port: 38657
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.16.124"
port: 43921
}
bound_http_address {
host: "127.24.16.124"
port: 44159
}
}
}
13:47:37.471 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:47:37.471 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:37.471 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:47:37.473 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:47:37.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.492708 24897 raft_consensus.cc:348] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:37.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.493378 24897 raft_consensus.cc:374] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:47:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.493712 24897 raft_consensus.cc:729] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 054d1028f8874ff39eecd59abc437469, State: Initialized, Role: FOLLOWER
13:47:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.494437 24897 consensus_queue.cc:260] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.497730 24908 sys_catalog.cc:455] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } } }
13:47:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.498661 24908 sys_catalog.cc:458] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [sys.catalog]: This master's current role is: FOLLOWER
13:47:37.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.499433 24897 sys_catalog.cc:564] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [sys.catalog]: configured and running, proceeding with master startup.
13:47:37.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139257, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:37.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:47:37.506 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:37.508 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc
13:47:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.515393 24851 tablet.cc:2369] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:47:37.516 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:47:37.517 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc
13:47:37.520 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:47:37.524 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
13:47:37.525 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:47:37.526 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:47:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.532778 24919 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.533017 24919 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:37.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:37 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139257, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:37.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:47:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.528481 24821 master_runner.cc:186] Error getting master registration for 127.24.16.126:35785: OK, instance_id { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" instance_seqno: 1714139254073636 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:47:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.559909 24821 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:37.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:37.564 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc
13:47:37.565 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:47:37.566 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.16.125:34507
13:47:37.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
13:47:37.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.568262 24641 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 24743
13:47:37.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:37.587 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.16.126:35785
13:47:37.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
13:47:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.587779 24641 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 24667
13:47:37.610 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:37.610 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.16.124:43921
13:47:37.610 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
13:47:37.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.611239 24641 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 24821
13:47:37.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:37.636 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.16.125:34507
13:47:37.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
13:47:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.639639 24641 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data
13:47:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/logs
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/info.pb
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.125:34507
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.125
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38657
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.125.keytab
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.125
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:47:37.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:47:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.912070 24923 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.912544 24923 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:37.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.913141 24923 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.940368 24923 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:47:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.940534 24923 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.940768 24923 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.940994 24923 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.941201 24923 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.941416 24923 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.941586 24923 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46665/jwks.json
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.971170 24923 master_runner.cc:386] Master server non-default flags:
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.125.keytab
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.125
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.125:34507
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/info.pb
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.125
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38657
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/logs
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.972319 24923 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:37.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.973083 24923 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.987020 24930 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:37.988296 24923 server_base.cc:1034] running on GCE node
13:47:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.988056 24931 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:37.987520 24933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.029081 24923 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:39.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.032184 24923 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:39.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.033543 24923 hybrid_clock.cc:648] HybridClock initialized: now 1714139259033507 us; error 46 us; skew 500 ppm
13:47:39.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.034265 24923 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:39 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139259, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.045998 24923 init.cc:377] Logged in from keytab as kudu/127.24.16.125@KRBTEST.COM (short username kudu)
13:47:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.050545 24923 webserver.cc:460] Webserver started at http://127.24.16.125:38657/ using document root <none> and password file <none>
13:47:39.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.051326 24923 fs_manager.cc:359] Metadata directory not provided
13:47:39.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.051497 24923 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:39.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.058230 24923 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.005s sys 0.002s
13:47:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.062255 24942 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.063140 24923 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.063390 24923 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f9c789a9f0e41d3adc7f0dbb5114073"
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:35 on dist-test-slave-2jsl"
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.065369 24923 fs_report.cc:383] FS layout report
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/data
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.117830 24923 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:39.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.122438 24923 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:39.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.122789 24923 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.190513 24923 rpc_server.cc:307] RPC server started. Bound to: 127.24.16.125:34507
13:47:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.190623 24993 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.16.125:34507 every 8 connection(s)
13:47:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.193296 24923 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/data/info.pb
13:47:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.200623 24641 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24923
13:47:39.201 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:39.201 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.16.126:35785
13:47:39.201 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
13:47:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.202678 24994 sys_catalog.cc:263] Verifying existing consensus state
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.202867 24641 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/logs
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/info.pb
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.126:35785
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.126
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34261
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.126.keytab
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.126
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:47:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240426 13:47:39.203414 24994 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:47:39.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.203730 24923 master.cc:537] Master@127.24.16.125:34507 shutting down...
13:47:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.267176 24923 master.cc:552] Master@127.24.16.125:34507 shutdown complete.
13:47:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:47:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.506704 24996 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.507151 24996 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.507655 24996 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.534446 24996 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:47:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.534621 24996 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.534860 24996 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.535035 24996 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.535223 24996 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.535460 24996 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.535611 24996 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46665/jwks.json
13:47:39.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.566481 24996 master_runner.cc:386] Master server non-default flags:
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.126.keytab
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.126
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.126:35785
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/info.pb
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.126
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34261
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/logs
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.567581 24996 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.568612 24996 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.582448 25004 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.583352 25005 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:39.584054 24996 server_base.cc:1034] running on GCE node
13:47:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:39.584830 25007 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.623351 24996 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:40.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.625828 24996 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:40.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.627187 24996 hybrid_clock.cc:648] HybridClock initialized: now 1714139260627165 us; error 33 us; skew 500 ppm
13:47:40.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.627960 24996 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:40 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139260, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:40.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.633424 24996 init.cc:377] Logged in from keytab as kudu/127.24.16.126@KRBTEST.COM (short username kudu)
13:47:40.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.638093 24996 webserver.cc:460] Webserver started at http://127.24.16.126:34261/ using document root <none> and password file <none>
13:47:40.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.639022 24996 fs_manager.cc:359] Metadata directory not provided
13:47:40.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.639215 24996 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:40.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.646392 24996 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.007s sys 0.000s
13:47:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.650189 25015 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.651047 24996 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.003s
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.651305 24996 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f316ad4a477145778a0330804f4c18f9"
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:33 on dist-test-slave-2jsl"
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.653213 24996 fs_report.cc:383] FS layout report
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/data
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:40.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.703649 24996 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.708557 24996 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:40.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.708922 24996 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:40.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.780086 24996 rpc_server.cc:307] RPC server started. Bound to: 127.24.16.126:35785
13:47:40.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.780174 25066 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.16.126:35785 every 8 connection(s)
13:47:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.782518 24996 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/data/info.pb
13:47:40.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.790927 24641 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 24996
13:47:40.792 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:40.792 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.16.124:43921
13:47:40.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.791762 25067 sys_catalog.cc:263] Verifying existing consensus state
13:47:40.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240426 13:47:40.792524 25067 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:47:40.792 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
13:47:40.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.792907 24996 master.cc:537] Master@127.24.16.126:35785 shutting down...
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.794191 24641 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/logs
13:47:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/info.pb
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.124:43921
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.124
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44159
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.124.keytab
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.124
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:47:40.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:40.848160 24996 master.cc:552] Master@127.24.16.126:35785 shutdown complete.
13:47:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:47:41.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.076463 25069 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.076928 25069 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:41.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.077399 25069 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.103796 25069 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:47:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.104046 25069 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.104416 25069 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.104737 25069 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.105024 25069 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.105307 25069 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.105448 25069 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46665/jwks.json
13:47:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:41.135345 25069 master_runner.cc:386] Master server non-default flags:
13:47:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.16.84:37003
13:47:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data
13:47:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.16.126:35785,127.24.16.125:34507,127.24.16.124:43921
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/krb5kdc/kudu_127.24.16.124.keytab
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.16.124
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.16.124:43921
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46665/jwks.json
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/info.pb
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.16.124
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44159
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/logs
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:41.136250 25069 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:41.137099 25069 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:47:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.152796 25077 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:41.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.571410 24485 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0060 after lost signal to thread 24360
13:47:41.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.572163 24485 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8000 after lost signal to thread 24488
13:47:42.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.153424 25076 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:42.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:41.153618 25079 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:42.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:41.154348 25069 server_base.cc:1034] running on GCE node
13:47:42.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.206804 25069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.210275 25069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:42.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.211719 25069 hybrid_clock.cc:648] HybridClock initialized: now 1714139262211730 us; error 125 us; skew 500 ppm
13:47:42.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.212872 25069 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:42 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139262, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.219300 25069 init.cc:377] Logged in from keytab as kudu/127.24.16.124@KRBTEST.COM (short username kudu)
13:47:42.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.224310 25069 webserver.cc:460] Webserver started at http://127.24.16.124:44159/ using document root <none> and password file <none>
13:47:42.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.225106 25069 fs_manager.cc:359] Metadata directory not provided
13:47:42.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.225282 25069 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.233105 25069 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
13:47:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.237433 25088 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.238461 25069 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.003s
13:47:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.238790 25069 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data,/tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "054d1028f8874ff39eecd59abc437469"
13:47:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:47:37 on dist-test-slave-2jsl"
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.241792 25069 fs_report.cc:383] FS layout report
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/wal
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/data
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:42.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.310771 25069 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:47:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.315905 25069 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:42.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.316350 25069 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:42.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.404132 25069 rpc_server.cc:307] RPC server started. Bound to: 127.24.16.124:43921
13:47:42.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.404320 25139 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.16.124:43921 every 8 connection(s)
13:47:42.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.406944 25069 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpcmmaJ/test-tmp/mini-kudu-cluster749691261735031946/master-2/data/info.pb
13:47:42.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.411873 24641 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu as pid 25069
13:47:42.412 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:42.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.420812 25140 sys_catalog.cc:263] Verifying existing consensus state
13:47:42.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.428277 25140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Bootstrap starting.
13:47:42.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.474347 25140 log.cc:824] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Log is configured to *not* fsync() on all Append() calls
13:47:42.475 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.484196 25140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:47:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.485219 25140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Bootstrap complete.
13:47:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.532490 25140 raft_consensus.cc:348] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.535938 25140 raft_consensus.cc:374] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:47:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.536324 25140 raft_consensus.cc:729] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 054d1028f8874ff39eecd59abc437469, State: Initialized, Role: FOLLOWER
13:47:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.537482 25140 consensus_queue.cc:260] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:42.552 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:42.554 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.16.124:43921(127.24.16.124:43921)] connecting to peer
13:47:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.554093 25145 sys_catalog.cc:455] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } } }
13:47:42.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.554944 25145 sys_catalog.cc:458] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [sys.catalog]: This master's current role is: FOLLOWER
13:47:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.556967 25140 sys_catalog.cc:564] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [sys.catalog]: configured and running, proceeding with master startup.
13:47:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:42.571811 25094 tablet.cc:2369] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:47:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.599422 25158 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
13:47:42.623 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.624 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.627 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.628 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.16.124:43921(127.24.16.124:43921)] Successfully connected to peer
13:47:42.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.617007 25069 master_runner.cc:186] Error getting master registration for 127.24.16.126:35785: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111),
13:47:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.648140 25069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:42.720 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
13:47:42.727 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.24.16.124
13:47:42.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:42 dist-test-slave-2jsl krb5kdc[24653](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139257, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.16.124@KRBTEST.COM
13:47:42.932 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3ea431af, L:/127.0.0.1:57752 - R:/127.24.16.124:43921] using SASL/GSSAPI
13:47:42.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.948647 25145 raft_consensus.cc:482] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:42.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.949120 25145 raft_consensus.cc:504] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:42.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:42.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.952380 25145 leader_election.cc:288] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785), 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507)
13:47:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:42.955245 25090 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:42.960 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:42.959537 25091 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785): Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111)
13:47:42.961 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:42.960247 25090 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507): Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.960618 25090 leader_election.cc:302] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 054d1028f8874ff39eecd59abc437469; no voters: 9f9c789a9f0e41d3adc7f0dbb5114073, f316ad4a477145778a0330804f4c18f9
13:47:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:42.961330 25145 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:42.966 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.967 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.968 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.968 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.968 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:42.978 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:42.978 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:42.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:42.983 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.984 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.984 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:42.984 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:42.998 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:42.999 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:43.001 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.003 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.003 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.004 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:43.038 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:43.040 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:43.041 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.042 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.046 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.046 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.047 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.047 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:43.078 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:43.079 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:43.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.082 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.083 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:43.158 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:43.159 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:43.161 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.161 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.161 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.163 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.164 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:43.178 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:43.180 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:43.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.182 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.182 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.182 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.183 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.184 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:43.439 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:43.440 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:43.441 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.442 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.443 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.445 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:43.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:43.617722 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:43.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:43.617952 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:43.638 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:43.640 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:43.640 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.641 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.641 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.643 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.643 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.644 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:43.778 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:43.780 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:43.780 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.781 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.781 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:43.784 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.784 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.784 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:43.784 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:44.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:44.620664 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:44.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:44.620903 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:44.692900 25165 raft_consensus.cc:482] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:44.693213 25165 raft_consensus.cc:504] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:44.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:44.694679 25165 leader_election.cc:288] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785), 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507)
13:47:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:44.701470 25090 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507): Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:44.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:44.702133 25091 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785): Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111)
13:47:44.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:44.702405 25091 leader_election.cc:302] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 054d1028f8874ff39eecd59abc437469; no voters: 9f9c789a9f0e41d3adc7f0dbb5114073, f316ad4a477145778a0330804f4c18f9
13:47:44.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:44.702889 25165 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:44.978 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:44.979 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:44.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:44.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:44.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:44.983 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:44.983 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:44.983 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:44.984 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:45.418 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:45.419 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:45.420 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:45.420 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:45.421 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:45.422 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:45.423 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:45.423 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:45.424 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:45.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:45.623453 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:45.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:45.623675 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:46.398 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:46.399 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:46.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:46.400 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:46.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:46.402 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:46.402 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:46.402 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:46.404 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:46.626317 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:46.626535 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:47.153800 25170 raft_consensus.cc:482] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:47.154134 25170 raft_consensus.cc:504] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:47.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:47.155400 25170 leader_election.cc:288] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785), 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507)
13:47:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:47.160084 25091 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785): Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111)
13:47:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:47.160480 25090 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507): Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:47.160907 25090 leader_election.cc:302] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 054d1028f8874ff39eecd59abc437469; no voters: 9f9c789a9f0e41d3adc7f0dbb5114073, f316ad4a477145778a0330804f4c18f9
13:47:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:47.161485 25170 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:47.629215 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:47.629515 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:47.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:47.650032 25069 master_runner.cc:186] Error getting master registration for 127.24.16.126:35785: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111),
13:47:47.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:47.650338 25069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:48.378 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:48.379 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:48.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:48.381 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:48.381 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:48.382 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:48.382 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:48.382 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:48.383 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:48.458 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:48.459 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:48.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:48.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:48.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:48.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:48.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:48.462 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:48.463 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:48.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:48.632184 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:48.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:48.632413 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:49.318 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:49.319 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:49.320 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:49.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:49.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:49.321 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:49.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:49.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:49.323 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:49.634994 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:49.635236 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:50.358 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:50.359 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:50.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:50.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:50.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:50.361 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:50.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:50.362 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:50.363 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:50.530786 25179 raft_consensus.cc:482] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:50.531143 25179 raft_consensus.cc:504] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:50.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:50.532773 25179 leader_election.cc:288] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785), 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507)
13:47:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:50.534222 25091 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111) [suppressed 5 similar messages]
13:47:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:50.536455 25091 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785): Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111)
13:47:50.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:50.536810 25090 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507): Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:50.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:50.537065 25090 leader_election.cc:302] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 054d1028f8874ff39eecd59abc437469; no voters: 9f9c789a9f0e41d3adc7f0dbb5114073, f316ad4a477145778a0330804f4c18f9
13:47:50.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:50.537520 25179 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:50.637703 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:50.637926 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:51.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:51.641469 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:51.641791 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:52.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:47:52 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139272, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:52.222899 25084 init.cc:295] Successfully reacquired a new kerberos TGT
13:47:52.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:52.644335 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:52.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:52.644556 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:52.654234 25069 master_runner.cc:186] Error getting master registration for 127.24.16.126:35785: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111),
13:47:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:52.654637 25069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:53.647176 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:53.647389 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:53.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:53.839 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:53.840 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:53.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:53.841 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:53.842 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:53.843 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:53.843 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:53.845 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:54.226861 25190 raft_consensus.cc:482] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:54.227218 25190 raft_consensus.cc:504] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:47:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:54.228900 25190 leader_election.cc:288] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785), 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507)
13:47:54.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:54.234181 25091 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785): Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111)
13:47:54.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:54.234705 25090 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507): Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:47:54.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:54.234969 25090 leader_election.cc:302] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 054d1028f8874ff39eecd59abc437469; no voters: 9f9c789a9f0e41d3adc7f0dbb5114073, f316ad4a477145778a0330804f4c18f9
13:47:54.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:54.235474 25190 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:54.649832 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:54.650076 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:55.652565 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:55.652792 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:56.418 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:56.419 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:56.420 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:56.420 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:56.421 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:56.421 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:56.421 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:56.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:56.423 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:56.655236 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:56.655464 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:57.138 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:47:57.139 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:47:57.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:57.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:57.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:47:57.141 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:57.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:57.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:47:57.143 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:47:57.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:57.657426 25069 master_runner.cc:186] Error getting master registration for 127.24.16.126:35785: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111),
13:47:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:47:57.657824 25069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:57.658442 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:57.658658 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:58.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:58.661140 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:58.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:58.661374 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:59.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:59.663900 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:59.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:47:59.664142 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:00.503803 25203 raft_consensus.cc:482] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:48:00.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:00.504146 25203 raft_consensus.cc:504] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:48:00.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:00.505496 25203 leader_election.cc:288] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785), 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507)
13:48:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:00.506942 25091 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111) [suppressed 3 similar messages]
13:48:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:00.509232 25091 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785): Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111)
13:48:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:00.509510 25090 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507): Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:48:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:00.509769 25090 leader_election.cc:302] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 054d1028f8874ff39eecd59abc437469; no voters: 9f9c789a9f0e41d3adc7f0dbb5114073, f316ad4a477145778a0330804f4c18f9
13:48:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:00.510251 25203 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:48:00.579 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:48:00.579 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:48:00.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:00.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:00.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:00.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:00.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:00.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:00.583 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:48:00.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:00.666836 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:00.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:00.667050 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:01.671025 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:01.671332 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:48:02 dist-test-slave-2jsl krb5kdc[24653](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714139282, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.16.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:48:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:02.224882 25084 init.cc:295] Successfully reacquired a new kerberos TGT
13:48:02.558 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:48:02.559 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:48:02.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:02.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:02.560 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:02.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:02.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:02.561 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:02.563 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:48:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:02.660632 25069 master_runner.cc:186] Error getting master registration for 127.24.16.126:35785: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111),
13:48:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:02.661034 25069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:48:02.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:02.673800 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:02.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:02.674022 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:03.138 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:48:03.139 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:48:03.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:03.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:03.140 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:03.141 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:03.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:03.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:03.143 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:48:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:03.452872 25213 raft_consensus.cc:482] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:48:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:03.453246 25213 raft_consensus.cc:504] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:48:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:03.454526 25213 leader_election.cc:288] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785), 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507)
13:48:03.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:03.458253 25091 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785): Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111)
13:48:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:03.458824 25090 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507): Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:48:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:03.459112 25090 leader_election.cc:302] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 054d1028f8874ff39eecd59abc437469; no voters: 9f9c789a9f0e41d3adc7f0dbb5114073, f316ad4a477145778a0330804f4c18f9
13:48:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:03.459630 25213 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:48:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:03.676785 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:03.677035 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:04.679919 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:04.680164 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:04.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:48:04.839 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:48:04.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:04.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:04.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:04.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:04.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:04.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:04.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:48:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:05.578019 25220 raft_consensus.cc:482] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:48:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:05.578357 25220 raft_consensus.cc:504] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f316ad4a477145778a0330804f4c18f9" member_type: VOTER last_known_addr { host: "127.24.16.126" port: 35785 } } peers { permanent_uuid: "9f9c789a9f0e41d3adc7f0dbb5114073" member_type: VOTER last_known_addr { host: "127.24.16.125" port: 34507 } } peers { permanent_uuid: "054d1028f8874ff39eecd59abc437469" member_type: VOTER last_known_addr { host: "127.24.16.124" port: 43921 } }
13:48:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:05.579800 25220 leader_election.cc:288] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785), 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507)
13:48:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:05.581584 25091 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111) [suppressed 3 similar messages]
13:48:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:05.584162 25090 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f9c789a9f0e41d3adc7f0dbb5114073 (127.24.16.125:34507): Network error: Client connection negotiation failed: client connection to 127.24.16.125:34507: connect: Connection refused (error 111)
13:48:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:05.584538 25091 leader_election.cc:334] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f316ad4a477145778a0330804f4c18f9 (127.24.16.126:35785): Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111)
13:48:05.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:05.584862 25091 leader_election.cc:302] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 054d1028f8874ff39eecd59abc437469; no voters: 9f9c789a9f0e41d3adc7f0dbb5114073, f316ad4a477145778a0330804f4c18f9
13:48:05.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:05.585379 25220 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:48:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:05.682860 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:05.683075 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:06.685384 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:06.685590 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:07.663311 25069 master_runner.cc:186] Error getting master registration for 127.24.16.126:35785: Network error: Client connection negotiation failed: client connection to 127.24.16.126:35785: connect: Connection refused (error 111),
13:48:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:07.663638 25069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:48:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:07.687945 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:07.688135 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:08.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:08.690389 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:08.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:08.690619 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:08.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:48:08.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:48:08.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:08.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:08.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:08.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:08.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:08.840 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:08.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:48:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:09.693116 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:09.693326 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:10.695638 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:10.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:10.695844 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:11.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:11.698899 25158 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 054d1028f8874ff39eecd59abc437469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:48:11.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:48:11.699188 25158 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:48:11.819 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.125:34507(127.24.16.125:34507)] connecting to peer
13:48:11.819 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.24.16.126:35785(127.24.16.126:35785)] connecting to peer
13:48:11.819 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:11.820 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.16.125:34507(127.24.16.125:34507)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:11.820 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.125:34507: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507
13:48:11.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:11.820 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.126:35785(127.24.16.126:35785)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:11.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.16.126:35785: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
13:48:11.823 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921; will retry
13:48:11.827 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29406), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [202ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [204ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [524ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [536ms] refreshing cache from master, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [544ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [546ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [547ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [558ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [579ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [581ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [585ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [619ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [626ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [662ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [744ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [759ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [760ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [762ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [762ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1016ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1216ms] refreshing cache from master, [1216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1219ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1224ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1358ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1362ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2556ms] refreshing cache from master, [2556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2559ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [2562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2996ms] refreshing cache from master, [2996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2998ms] too many traces: truncated at 100 traces, deferred=Deferred@1485215725(state=PENDING, result=null, callback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515), errback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515)))
13:48:11.831 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.16.125:34507(127.24.16.125:34507)] disconnecting while in state TERMINATED
13:48:11.833 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.16.124:43921(127.24.16.124:43921)] disconnecting while in state READY
13:48:11.833 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.16.126:35785(127.24.16.126:35785)] disconnecting while in state TERMINATED
13:48:11.836 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:48:11.838 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.16.124:43921(127.24.16.124:43921)] handling channelInactive
13:48:11.838 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.16.124:43921(127.24.16.124:43921)] cleaning up while in state READY due to: connection closed
13:48:11.840 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:48:11.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:11.844612 24641 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 25069
13:48:12.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:48:12Z chronyd exiting
13:48:13.884 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:48:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:13.885810 23920 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 24201
13:48:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:13.909401 23920 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 24354
13:48:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:13.940369 23920 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 24499
13:48:13.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:13.963054 23920 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 23932
13:48:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:14.000945 23920 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 24002
13:48:14.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:48:14.027033 23920 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpcmmaJ/build/tsan/bin/kudu with pid 24077
13:48:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:48:14Z chronyd exiting
13:48:16.035 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29406), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [202ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [204ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [524ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [536ms] refreshing cache from master, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [544ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [546ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [547ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [558ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [579ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [581ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [585ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [619ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [626ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [662ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [744ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [759ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [760ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [762ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [762ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1016ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1216ms] refreshing cache from master, [1216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1219ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1224ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1358ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1362ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2556ms] refreshing cache from master, [2556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2559ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [2562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2996ms] refreshing cache from master, [2996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2998ms] too many traces: truncated at 100 traces, deferred=Deferred@1485215725(state=PENDING, result=null, callback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515), errback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.16.126:35785
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29406), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [202ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [204ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [524ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [536ms] refreshing cache from master, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [544ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [546ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [547ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [558ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [579ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [581ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [585ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [619ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [626ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [662ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [744ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [759ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [760ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [762ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [762ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1016ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1216ms] refreshing cache from master, [1216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1219ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1224ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1358ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1362ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2556ms] refreshing cache from master, [2556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2559ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [2562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2996ms] refreshing cache from master, [2996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2998ms] too many traces: truncated at 100 traces, deferred=Deferred@1485215725(state=PENDING, result=null, callback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515), errback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.16.126:35785
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13:48:16.480 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
13:48:16.480 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29406), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [202ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [204ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [524ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [536ms] refreshing cache from master, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [544ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [546ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [547ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [558ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [579ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [581ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [585ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [619ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [626ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [662ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [744ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [759ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [760ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [762ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [762ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1016ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1216ms] refreshing cache from master, [1216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1219ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1224ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1358ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1362ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2556ms] refreshing cache from master, [2556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2559ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [2562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2996ms] refreshing cache from master, [2996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2998ms] too many traces: truncated at 100 traces, deferred=Deferred@1485215725(state=PENDING, result=null, callback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515), errback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.16.126:35785
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29406), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [202ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [204ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [524ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [529ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [536ms] refreshing cache from master, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [544ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [546ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [547ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [558ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [579ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [581ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [583ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [585ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [619ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [623ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [626ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [659ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [662ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [663ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [739ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [741ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [743ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [744ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [759ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [760ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [762ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [762ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1016ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1216ms] refreshing cache from master, [1216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1219ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1224ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [1358ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [1362ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2556ms] refreshing cache from master, [2556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2559ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.125:34507: Network error: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.126:35785: Network error: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2562ms] Sub RPC ConnectToMaster: received response from server master-127.24.16.124:43921: OK, [2562ms] delaying RPC due to: Service unavailable: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785, [2996ms] refreshing cache from master, [2996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.125:34507, [2997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.126:35785, [2998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.16.124:43921, [2998ms] too many traces: truncated at 100 traces, deferred=Deferred@1485215725(state=PENDING, result=null, callback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515), errback=(continuation of Deferred@253499196 after retry RPC after error@338943539) -> (continuation of Deferred@942261983 after retry RPC after error@1417414600) -> (continuation of Deferred@1001807756 after retry RPC after error@838786191) -> (continuation of Deferred@1201138513 after retry RPC after error@1935799852) -> (continuation of Deferred@2057473523 after retry RPC after error@1040133260) -> (continuation of Deferred@1162515713 after retry RPC after error@971426578) -> (continuation of Deferred@1472529608 after retry RPC after error@320526832) -> (continuation of Deferred@40477711 after retry RPC after error@223891723) -> (continuation of Deferred@1925510956 after retry RPC after error@936670685) -> (continuation of Deferred@2018703092 after retry RPC after error@927801596) -> (continuation of Deferred@1650842798 after retry RPC after error@1980442530) -> (continuation of Deferred@419531457 after retry RPC after error@1200948189) -> (continuation of Deferred@198784813 after retry RPC after error@608188355) -> (continuation of Deferred@822277640 after retry RPC after error@1592164382) -> (continuation of Deferred@474260656 after retry RPC after error@958363348) -> (continuation of Deferred@520840521 after retry RPC after error@1963454662) -> (continuation of Deferred@1281338354 after retry RPC after error@246730107) -> (continuation of Deferred@1255264457 after retry RPC after error@10127509) -> (continuation of Deferred@833336143 after retry RPC after error@425492244) -> (continuation of Deferred@1565504366 after retry RPC after error@155534962) -> (continuation of Deferred@1696116190 after retry RPC after error@524733110) -> (continuation of Deferred@1068687091 after retry RPC after error@476543393) -> (continuation of Deferred@1009861038 after retry RPC after error@353058788) -> (continuation of Deferred@1021783792 after retry RPC after error@468382396) -> (continuation of Deferred@769715294 after retry RPC after error@1641550361) -> (continuation of Deferred@1682776188 after retry RPC after error@687751515)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.24.16.125:34507,127.24.16.126:35785,127.24.16.124:43921) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.125:34507(127.24.16.125:34507): Connection refused: /127.24.16.125:34507,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.24.16.126:35785(127.24.16.126:35785): Connection refused: /127.24.16.126:35785
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.24.16.126:35785
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)