14:19:28.964 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:19:28.975 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taske_AVaV/test-tmp
14:19:28.977 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
14:19:28.980 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taske_AVaV/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
14:19:29.240 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:19:29.389 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:19:29.390 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:19:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.393785 29867 mini_oidc.cc:152] Starting JWKS server
14:19:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.397010 29867 webserver.cc:460] Webserver started at http://127.0.0.1:46337/ using document root <none> and password file <none>
14:19:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.397287 29867 mini_oidc.cc:189] Starting OIDC Discovery server
14:19:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.397965 29867 webserver.cc:460] Webserver started at http://127.0.0.1:33569/ using document root <none> and password file <none>
14:19:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:19:29Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
14:19:29.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:19:29Z Disabled control of system clock
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.428802 29867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/wal
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/data
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/logs
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/data/info.pb
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.254:35087
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.254
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46337/jwks.json with env {}
14:19:29.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.732723 29878 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:29.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.732995 29878 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:29.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.733089 29878 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.740722 29878 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.740840 29878 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.740883 29878 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.740940 29878 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.740976 29878 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.741019 29878 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46337/jwks.json
14:19:29.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.750528 29878 master_runner.cc:386] Master server non-default flags:
14:19:29.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/data
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/wal
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.254:35087
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46337/jwks.json
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/data/info.pb
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.254
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/logs
14:19:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.752130 29878 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.754384 29878 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:29.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.765228 29883 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:29.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.765717 29884 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.766292 29886 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.766580 29878 server_base.cc:1034] running on GCE node
14:19:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.767320 29878 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:29.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.768631 29878 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:29.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.769882 29878 hybrid_clock.cc:648] HybridClock initialized: now 1714659569769789 us; error 52 us; skew 500 ppm
14:19:29.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.770265 29878 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:29.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.778328 29878 webserver.cc:460] Webserver started at http://127.29.42.254:40783/ using document root <none> and password file <none>
14:19:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.779076 29878 fs_manager.cc:359] Metadata directory not provided
14:19:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.779213 29878 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.779608 29878 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.782260 29878 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/data/instance:
14:19:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1333a1b2d984588a283cad84b4a7b46"
14:19:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:29 on dist-test-slave-q1bb"
14:19:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.783183 29878 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/wal/instance:
14:19:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1333a1b2d984588a283cad84b4a7b46"
14:19:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:29 on dist-test-slave-q1bb"
14:19:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.788300 29878 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
14:19:29.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.791649 29892 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.792914 29878 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:19:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.793088 29878 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/wal
14:19:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1333a1b2d984588a283cad84b4a7b46"
14:19:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:29 on dist-test-slave-q1bb"
14:19:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.793314 29878 fs_report.cc:383] FS layout report
14:19:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/wal
14:19:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/wal
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/data/data
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:29.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.816329 29878 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:29.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.817253 29878 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:29.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.817593 29878 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.835201 29878 rpc_server.cc:307] RPC server started. Bound to: 127.29.42.254:35087
14:19:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.835238 29943 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.42.254:35087 every 8 connection(s)
14:19:29.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.836622 29878 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/data/info.pb
14:19:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.840404 29944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.843248 29867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 29878
14:19:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.843467 29867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-0/wal/instance
14:19:29.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.845435 29867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:29.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:29.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/wal
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/data
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/logs
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/data/info.pb
14:19:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.253:38615
14:19:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.253
14:19:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:29.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:29.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:29.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46337/jwks.json with env {}
14:19:29.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.846643 29944 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } has no permanent_uuid. Determining permanent_uuid...
14:19:29.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.864688 29944 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } has no permanent_uuid. Determining permanent_uuid...
14:19:29.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.865869 29896 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.42.253:38615: connect: Connection refused (error 111)
14:19:29.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.867353 29944 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.253:38615: Network error: Client connection negotiation failed: client connection to 127.29.42.253:38615: connect: Connection refused (error 111)
14:19:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.916643 29944 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } attempt: 1
14:19:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.918561 29944 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.253:38615: Network error: Client connection negotiation failed: client connection to 127.29.42.253:38615: connect: Connection refused (error 111)
14:19:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:29.986884 29944 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } attempt: 2
14:19:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:29.988760 29944 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.253:38615: Network error: Client connection negotiation failed: client connection to 127.29.42.253:38615: connect: Connection refused (error 111)
14:19:30.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.080003 29944 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } attempt: 3
14:19:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.081750 29944 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.253:38615: Network error: Client connection negotiation failed: client connection to 127.29.42.253:38615: connect: Connection refused (error 111)
14:19:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.180256 29946 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.180570 29946 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.180723 29946 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.188114 29946 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.188231 29946 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.188292 29946 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.188355 29946 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.188405 29946 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:30.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.188462 29946 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46337/jwks.json
14:19:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.197944 29946 master_runner.cc:386] Master server non-default flags:
14:19:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/data
14:19:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/wal
14:19:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.253:38615
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46337/jwks.json
14:19:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/data/info.pb
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.253
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/logs
14:19:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:30.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:30.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:30.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.199700 29946 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.202067 29946 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:30.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.212850 29955 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:30.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.213513 29954 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.213748 29957 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.214028 29946 server_base.cc:1034] running on GCE node
14:19:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.214694 29946 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.215968 29946 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.217180 29946 hybrid_clock.cc:648] HybridClock initialized: now 1714659570217112 us; error 57 us; skew 500 ppm
14:19:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.217466 29946 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:30.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.219759 29946 webserver.cc:460] Webserver started at http://127.29.42.253:38027/ using document root <none> and password file <none>
14:19:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.220582 29946 fs_manager.cc:359] Metadata directory not provided
14:19:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.220731 29946 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.221076 29946 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.223802 29946 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/data/instance:
14:19:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c405bcd8ec2b4480b609bd30e32b598a"
14:19:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:30 on dist-test-slave-q1bb"
14:19:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.224634 29946 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/wal/instance:
14:19:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c405bcd8ec2b4480b609bd30e32b598a"
14:19:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:30 on dist-test-slave-q1bb"
14:19:30.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.225026 29944 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } attempt: 4
14:19:30.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.226680 29944 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.253:38615: Network error: Client connection negotiation failed: client connection to 127.29.42.253:38615: connect: Connection refused (error 111)
14:19:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.231133 29946 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
14:19:30.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.234505 29963 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.235703 29946 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.004s sys 0.000s
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.235838 29946 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/wal
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c405bcd8ec2b4480b609bd30e32b598a"
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:30 on dist-test-slave-q1bb"
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.236058 29946 fs_report.cc:383] FS layout report
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/wal
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/wal
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/data/data
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.257676 29946 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.258478 29946 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.258729 29946 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.276125 29946 rpc_server.cc:307] RPC server started. Bound to: 127.29.42.253:38615
14:19:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.276176 30014 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.42.253:38615 every 8 connection(s)
14:19:30.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.277390 29946 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/data/info.pb
14:19:30.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.280936 30015 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:30.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.284829 29867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 29946
14:19:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.285022 29867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-1/wal/instance
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.286255 29867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/wal
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/data
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/logs
14:19:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/data/info.pb
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.252:45293
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.252
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46337/jwks.json with env {}
14:19:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.287243 30015 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } has no permanent_uuid. Determining permanent_uuid...
14:19:30.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.301362 30015 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } has no permanent_uuid. Determining permanent_uuid...
14:19:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.309981 30015 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } has no permanent_uuid. Determining permanent_uuid...
14:19:30.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.310999 29965 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.312420 30015 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.252:45293: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.361728 30015 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } attempt: 1
14:19:30.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.363724 30015 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.252:45293: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.431941 30015 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } attempt: 2
14:19:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.434703 30015 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.252:45293: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.525935 30015 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } attempt: 3
14:19:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.526034 29944 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } attempt: 5
14:19:30.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.528934 30015 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.252:45293: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.533004 29944 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } has no permanent_uuid. Determining permanent_uuid...
14:19:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.534534 29944 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.252:45293: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.585842 29944 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } attempt: 1
14:19:30.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.587695 29944 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.252:45293: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.605474 30017 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:30.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.605799 30017 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:30.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.605970 30017 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.613783 30017 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.613941 30017 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.614009 30017 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.614082 30017 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.614143 30017 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.614212 30017 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46337/jwks.json
14:19:30.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.623548 30017 master_runner.cc:386] Master server non-default flags:
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/data
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/wal
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.252:45293
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46337/jwks.json
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/data/info.pb
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.252
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/logs
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.625309 30017 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.627612 30017 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.638576 30025 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:30.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.639056 30026 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:30.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.639637 30028 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:30.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.639919 30017 server_base.cc:1034] running on GCE node
14:19:30.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.640734 30017 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.642144 30017 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.643358 30017 hybrid_clock.cc:648] HybridClock initialized: now 1714659570643291 us; error 48 us; skew 500 ppm
14:19:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.643671 30017 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.646047 30017 webserver.cc:460] Webserver started at http://127.29.42.252:34571/ using document root <none> and password file <none>
14:19:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.646683 30017 fs_manager.cc:359] Metadata directory not provided
14:19:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.646806 30017 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.647078 30017 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.649600 30017 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/data/instance:
14:19:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5766b19f4964424aae4d9474984e1b79"
14:19:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:30 on dist-test-slave-q1bb"
14:19:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.650496 30017 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/wal/instance:
14:19:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5766b19f4964424aae4d9474984e1b79"
14:19:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:30 on dist-test-slave-q1bb"
14:19:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.655718 30017 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.001s
14:19:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.655912 29944 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } attempt: 2
14:19:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.657831 29944 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.252:45293: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.659479 30034 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:30.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.660713 30017 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.660882 30017 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/wal
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5766b19f4964424aae4d9474984e1b79"
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:30 on dist-test-slave-q1bb"
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.661094 30017 fs_report.cc:383] FS layout report
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/wal
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/wal
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/data/data
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:30.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.672158 30015 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } attempt: 4
14:19:30.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.673828 30015 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.42.252:45293: Network error: Client connection negotiation failed: client connection to 127.29.42.252:45293: connect: Connection refused (error 111)
14:19:30.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.680760 30017 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.681671 30017 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.682031 30017 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.701517 30017 rpc_server.cc:307] RPC server started. Bound to: 127.29.42.252:45293
14:19:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.701552 30085 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.42.252:45293 every 8 connection(s)
14:19:30.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.703094 30017 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/data/info.pb
14:19:30.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.706991 30086 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:30.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.712739 29867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30017
14:19:30.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.712903 29867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/master-2/wal/instance
14:19:30.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.712721 30086 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } has no permanent_uuid. Determining permanent_uuid...
14:19:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.725630 30086 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } has no permanent_uuid. Determining permanent_uuid...
14:19:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.733165 30086 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } has no permanent_uuid. Determining permanent_uuid...
14:19:30.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.747150 30086 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79: Bootstrap starting.
14:19:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.750375 30086 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79: Neither blocks nor log segments found. Creating new log.
14:19:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.751354 30086 log.cc:824] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79: Log is configured to *not* fsync() on all Append() calls
14:19:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.753751 30086 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79: No bootstrap required, opened a new log
14:19:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.759850 30086 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.760125 30086 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:19:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.760211 30086 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5766b19f4964424aae4d9474984e1b79, State: Initialized, Role: FOLLOWER
14:19:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.760766 30086 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.761565 30095 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.762214 30095 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: This master's current role is: FOLLOWER
14:19:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.762650 30095 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:19:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.763034 30095 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.763238 30086 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: configured and running, proceeding with master startup.
14:19:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.764209 29944 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } attempt: 3
14:19:30.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.766829 30095 leader_election.cc:288] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d1333a1b2d984588a283cad84b4a7b46 (127.29.42.254:35087), c405bcd8ec2b4480b609bd30e32b598a (127.29.42.253:38615)
14:19:30.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.780213 29944 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46: Bootstrap starting.
14:19:30.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.785679 29944 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46: Neither blocks nor log segments found. Creating new log.
14:19:30.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.788939 29944 log.cc:824] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46: Log is configured to *not* fsync() on all Append() calls
14:19:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.805330 29944 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46: No bootstrap required, opened a new log
14:19:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.815135 29944 raft_consensus.cc:348] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.815438 29944 raft_consensus.cc:374] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:19:30.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.815532 29944 raft_consensus.cc:729] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d1333a1b2d984588a283cad84b4a7b46, State: Initialized, Role: FOLLOWER
14:19:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.816179 29944 consensus_queue.cc:260] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:30.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.809114 30017 master_runner.cc:186] Error getting master registration for 127.29.42.254:35087: OK, instance_id { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" instance_seqno: 1714659569830677 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:19:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.817102 30017 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.817553 30110 sys_catalog.cc:455] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.818087 30110 sys_catalog.cc:458] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [sys.catalog]: This master's current role is: FOLLOWER
14:19:30.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.819214 30108 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:30.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.819319 30108 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.821116 29944 sys_catalog.cc:564] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [sys.catalog]: configured and running, proceeding with master startup.
14:19:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.821169 29919 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5766b19f4964424aae4d9474984e1b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d1333a1b2d984588a283cad84b4a7b46" is_pre_election: true
14:19:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.821722 29919 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5766b19f4964424aae4d9474984e1b79 in term 0.
14:19:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.822417 30036 leader_election.cc:302] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5766b19f4964424aae4d9474984e1b79, d1333a1b2d984588a283cad84b4a7b46; no voters:
14:19:30.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.822808 30096 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:19:30.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.822949 30096 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:19:30.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.823071 30096 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 0 FOLLOWER]: Advancing to term 1
14:19:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.825938 30096 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.827240 29919 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5766b19f4964424aae4d9474984e1b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d1333a1b2d984588a283cad84b4a7b46"
14:19:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.827328 30096 leader_election.cc:288] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [CANDIDATE]: Term 1 election: Requested vote from peers d1333a1b2d984588a283cad84b4a7b46 (127.29.42.254:35087), c405bcd8ec2b4480b609bd30e32b598a (127.29.42.253:38615)
14:19:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.827471 29919 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [term 0 FOLLOWER]: Advancing to term 1
14:19:30.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.831534 29919 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5766b19f4964424aae4d9474984e1b79 in term 1.
14:19:30.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.832329 30036 leader_election.cc:302] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5766b19f4964424aae4d9474984e1b79, d1333a1b2d984588a283cad84b4a7b46; no voters:
14:19:30.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.832644 30096 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 1 FOLLOWER]: Leader election won for term 1
14:19:30.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.832837 30096 raft_consensus.cc:686] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [term 1 LEADER]: Becoming Leader. State: Replica: 5766b19f4964424aae4d9474984e1b79, State: Running, Role: LEADER
14:19:30.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.833413 30096 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.836387 30095 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5766b19f4964424aae4d9474984e1b79. Latest consensus state: current_term: 1 leader_uuid: "5766b19f4964424aae4d9474984e1b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.836654 30095 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: This master's current role is: LEADER
14:19:30.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.845583 30116 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
14:19:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.848212 30116 catalog_manager.cc:1484] Initializing Kudu cluster ID...
14:19:30.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.860740 29919 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [term 1 FOLLOWER]: Refusing update from remote peer 5766b19f4964424aae4d9474984e1b79: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:19:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.861752 30096 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:19:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.876545 30124 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:30.876683 30124 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.878372 30110 sys_catalog.cc:455] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5766b19f4964424aae4d9474984e1b79. Latest consensus state: current_term: 1 leader_uuid: "5766b19f4964424aae4d9474984e1b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.878667 30110 sys_catalog.cc:458] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [sys.catalog]: This master's current role is: FOLLOWER
14:19:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.880555 30120 mvcc.cc:205] Tried to move back new op lower bound from 7023245602233303040 to 7023245602138783744. Current Snapshot: MvccSnapshot[applied={T|T < 7023245602233303040}]
14:19:30.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.892093 30095 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5766b19f4964424aae4d9474984e1b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:30.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.892400 30095 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: This master's current role is: LEADER
14:19:30.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.893590 30116 catalog_manager.cc:1347] Generated new cluster ID: 413a7d26ba674a6681d54ef9ffd4eb1f
14:19:30.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.893702 30116 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
14:19:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.873299 29878 master_runner.cc:186] Error getting master registration for 127.29.42.253:38615: OK, instance_id { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" instance_seqno: 1714659570271229 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:19:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.906534 30110 sys_catalog.cc:455] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5766b19f4964424aae4d9474984e1b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.906787 30110 sys_catalog.cc:458] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46 [sys.catalog]: This master's current role is: FOLLOWER
14:19:30.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.905531 29878 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:30.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.925441 30116 catalog_manager.cc:1370] Generated new certificate authority record
14:19:30.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.927367 30116 catalog_manager.cc:1504] Loading token signing keys...
14:19:30.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.942436 30116 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79: Generated new TSK 0
14:19:30.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.944759 30116 catalog_manager.cc:1514] Initializing in-progress tserver states...
14:19:30.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.969733 29867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:30.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:30.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/wal
14:19:30.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/data
14:19:30.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/logs
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/data/info.pb
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.193:0
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.42.193
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.193
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:30.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:30.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:30.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:30.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:19:30.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.973047 30015 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } attempt: 5
14:19:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.988551 30015 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a: Bootstrap starting.
14:19:30.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.996008 30015 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a: Neither blocks nor log segments found. Creating new log.
14:19:30.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:30.998670 30015 log.cc:824] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a: Log is configured to *not* fsync() on all Append() calls
14:19:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.012565 30015 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a: No bootstrap required, opened a new log
14:19:31.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.028898 30015 raft_consensus.cc:348] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:31.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.029807 30015 raft_consensus.cc:374] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:19:31.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.030273 30015 raft_consensus.cc:729] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c405bcd8ec2b4480b609bd30e32b598a, State: Initialized, Role: FOLLOWER
14:19:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.031718 30015 consensus_queue.cc:260] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } }
14:19:31.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.043777 29990 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5766b19f4964424aae4d9474984e1b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c405bcd8ec2b4480b609bd30e32b598a" is_pre_election: true
14:19:31.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.044211 29989 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5766b19f4964424aae4d9474984e1b79" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c405bcd8ec2b4480b609bd30e32b598a"
14:19:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.045001 29990 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5766b19f4964424aae4d9474984e1b79 in term 0.
14:19:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.045904 29988 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [term 0 FOLLOWER]: Advancing to term 1
14:19:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.046319 30015 sys_catalog.cc:564] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [sys.catalog]: configured and running, proceeding with master startup.
14:19:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.049482 29988 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [term 1 FOLLOWER]: Refusing update from remote peer 5766b19f4964424aae4d9474984e1b79: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:19:31.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.050580 30132 sys_catalog.cc:455] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.050827 30095 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:19:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.051667 30132 sys_catalog.cc:458] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [sys.catalog]: This master's current role is: FOLLOWER
14:19:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.054832 29989 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5766b19f4964424aae4d9474984e1b79 in term 1.
14:19:31.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.104152 30132 sys_catalog.cc:455] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5766b19f4964424aae4d9474984e1b79. Latest consensus state: current_term: 1 leader_uuid: "5766b19f4964424aae4d9474984e1b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:31.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.104454 30132 sys_catalog.cc:458] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [sys.catalog]: This master's current role is: FOLLOWER
14:19:31.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.105098 30137 sys_catalog.cc:455] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5766b19f4964424aae4d9474984e1b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:31.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.106527 30137 sys_catalog.cc:458] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a [sys.catalog]: This master's current role is: FOLLOWER
14:19:31.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.152046 30148 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:19:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.468938 30130 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.469237 30130 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.469363 30130 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.476449 30130 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.476662 30130 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.42.193
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.485029 30130 tablet_server_runner.cc:78] Tablet server non-default flags:
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/data
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/wal
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.193:0
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/data/info.pb
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.193
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.42.193
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/logs
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.486634 30130 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.488337 30130 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.499469 30154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.499984 30153 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.500396 30156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.500638 30130 server_base.cc:1034] running on GCE node
14:19:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.501161 30130 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:31.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.502029 30130 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.503224 30130 hybrid_clock.cc:648] HybridClock initialized: now 1714659571503166 us; error 46 us; skew 500 ppm
14:19:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.503475 30130 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.505540 30130 webserver.cc:460] Webserver started at http://127.29.42.193:34519/ using document root <none> and password file <none>
14:19:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.506198 30130 fs_manager.cc:359] Metadata directory not provided
14:19:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.506307 30130 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.506592 30130 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.508860 30130 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/data/instance:
14:19:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b29ba095ab242d4b7372c99e4979f26"
14:19:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:31 on dist-test-slave-q1bb"
14:19:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.509577 30130 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/wal/instance:
14:19:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b29ba095ab242d4b7372c99e4979f26"
14:19:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:31 on dist-test-slave-q1bb"
14:19:31.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.514230 30130 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.000s sys 0.003s
14:19:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.517089 30162 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.518329 30130 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
14:19:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.518455 30130 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/wal
14:19:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b29ba095ab242d4b7372c99e4979f26"
14:19:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:31 on dist-test-slave-q1bb"
14:19:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.518584 30130 fs_report.cc:383] FS layout report
14:19:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/wal
14:19:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/wal
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/data/data
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.536919 30130 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.537662 30130 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:31.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.537932 30130 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:31.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.538743 30130 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
14:19:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.540237 30130 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:19:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.540306 30130 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:19:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.540383 30130 ts_tablet_manager.cc:590] Registered 0 tablets
14:19:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.540417 30130 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:19:31.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.569854 30130 rpc_server.cc:307] RPC server started. Bound to: 127.29.42.193:45053
14:19:31.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.569902 30274 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.42.193:45053 every 8 connection(s)
14:19:31.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.571044 30130 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/data/info.pb
14:19:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.576637 29867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30130
14:19:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.576900 29867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-0/wal/instance
14:19:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.583422 29867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/wal
14:19:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/data
14:19:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/logs
14:19:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/data/info.pb
14:19:31.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.194:0
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.42.194
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.194
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:19:31.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.602041 30275 heartbeater.cc:347] Connected to a master server at 127.29.42.252:45293
14:19:31.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.602380 30275 heartbeater.cc:464] Registering TS with master...
14:19:31.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.603475 30275 heartbeater.cc:511] Master 127.29.42.252:45293 requested a full tablet report, sending...
14:19:31.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.604005 30276 heartbeater.cc:347] Connected to a master server at 127.29.42.254:35087
14:19:31.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.604147 30276 heartbeater.cc:464] Registering TS with master...
14:19:31.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.604396 30277 heartbeater.cc:347] Connected to a master server at 127.29.42.253:38615
14:19:31.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.604560 30277 heartbeater.cc:464] Registering TS with master...
14:19:31.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.605139 30277 heartbeater.cc:511] Master 127.29.42.253:38615 requested a full tablet report, sending...
14:19:31.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.605571 30276 heartbeater.cc:511] Master 127.29.42.254:35087 requested a full tablet report, sending...
14:19:31.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.607053 30051 ts_manager.cc:194] Registered new tserver with Master: 4b29ba095ab242d4b7372c99e4979f26 (127.29.42.193:45053)
14:19:31.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.607046 29908 ts_manager.cc:194] Registered new tserver with Master: 4b29ba095ab242d4b7372c99e4979f26 (127.29.42.193:45053)
14:19:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.608717 30051 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.42.193:42279
14:19:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.610827 29980 ts_manager.cc:194] Registered new tserver with Master: 4b29ba095ab242d4b7372c99e4979f26 (127.29.42.193:45053)
14:19:31.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.615216 30095 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5766b19f4964424aae4d9474984e1b79" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d1333a1b2d984588a283cad84b4a7b46" member_type: VOTER last_known_addr { host: "127.29.42.254" port: 35087 } } peers { permanent_uuid: "c405bcd8ec2b4480b609bd30e32b598a" member_type: VOTER last_known_addr { host: "127.29.42.253" port: 38615 } } peers { permanent_uuid: "5766b19f4964424aae4d9474984e1b79" member_type: VOTER last_known_addr { host: "127.29.42.252" port: 45293 } } }
14:19:31.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.615783 30095 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 [sys.catalog]: This master's current role is: LEADER
14:19:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.879089 30124 catalog_manager.cc:1259] Loaded cluster ID: 413a7d26ba674a6681d54ef9ffd4eb1f
14:19:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.879212 30124 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46: loading cluster ID for follower catalog manager: success
14:19:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.883091 30124 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46: acquiring CA information for follower catalog manager: success
14:19:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.885555 30124 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d1333a1b2d984588a283cad84b4a7b46: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:19:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.919385 30283 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.919644 30283 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.919729 30283 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:31.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.926523 30283 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:31.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.926682 30283 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.42.194
14:19:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.935078 30283 tablet_server_runner.cc:78] Tablet server non-default flags:
14:19:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/data
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/wal
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.194:0
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/data/info.pb
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.194
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.42.194
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/logs
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:31.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.936367 30283 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.938364 30283 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:31.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.949425 30293 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.950044 30294 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:31.950400 30296 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.950620 30283 server_base.cc:1034] running on GCE node
14:19:31.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.951221 30283 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:31.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.952102 30283 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.953320 30283 hybrid_clock.cc:648] HybridClock initialized: now 1714659571953263 us; error 29 us; skew 500 ppm
14:19:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.953630 30283 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.955714 30283 webserver.cc:460] Webserver started at http://127.29.42.194:43143/ using document root <none> and password file <none>
14:19:31.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.956379 30283 fs_manager.cc:359] Metadata directory not provided
14:19:31.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.956516 30283 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:31.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.956821 30283 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:31.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.959262 30283 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/data/instance:
14:19:31.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e869f4131f254720aaf95cdea256e4ea"
14:19:31.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:31 on dist-test-slave-q1bb"
14:19:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.960068 30283 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/wal/instance:
14:19:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e869f4131f254720aaf95cdea256e4ea"
14:19:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:31 on dist-test-slave-q1bb"
14:19:31.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.964661 30283 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.001s
14:19:31.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.967828 30302 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.969009 30283 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
14:19:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.969168 30283 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/wal
14:19:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e869f4131f254720aaf95cdea256e4ea"
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:31 on dist-test-slave-q1bb"
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.969367 30283 fs_report.cc:383] FS layout report
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/wal
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/wal
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/data/data
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:31.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.988911 30283 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:31.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.989715 30283 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:31.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.990042 30283 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:31.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.990978 30283 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
14:19:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.992396 30283 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:19:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.992478 30283 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:19:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.992592 30283 ts_tablet_manager.cc:590] Registered 0 tablets
14:19:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:31.992650 30283 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:19:32.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.023324 30283 rpc_server.cc:307] RPC server started. Bound to: 127.29.42.194:33731
14:19:32.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.023363 30414 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.42.194:33731 every 8 connection(s)
14:19:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.024564 30283 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/data/info.pb
14:19:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.029691 29867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30283
14:19:32.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.029882 29867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-1/wal/instance
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.034173 29867 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/wal
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/data
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/logs
14:19:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/data/info.pb
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.195:0
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.42.195
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.195
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:19:32.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.044497 30415 heartbeater.cc:347] Connected to a master server at 127.29.42.252:45293
14:19:32.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.044749 30415 heartbeater.cc:464] Registering TS with master...
14:19:32.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.045643 30415 heartbeater.cc:511] Master 127.29.42.252:45293 requested a full tablet report, sending...
14:19:32.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.046160 30417 heartbeater.cc:347] Connected to a master server at 127.29.42.253:38615
14:19:32.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.046335 30417 heartbeater.cc:464] Registering TS with master...
14:19:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.046847 30417 heartbeater.cc:511] Master 127.29.42.253:38615 requested a full tablet report, sending...
14:19:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.047559 30051 ts_manager.cc:194] Registered new tserver with Master: e869f4131f254720aaf95cdea256e4ea (127.29.42.194:33731)
14:19:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.047868 29980 ts_manager.cc:194] Registered new tserver with Master: e869f4131f254720aaf95cdea256e4ea (127.29.42.194:33731)
14:19:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.048535 30051 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.42.194:48469
14:19:32.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.051297 30416 heartbeater.cc:347] Connected to a master server at 127.29.42.254:35087
14:19:32.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.051445 30416 heartbeater.cc:464] Registering TS with master...
14:19:32.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.051800 30416 heartbeater.cc:511] Master 127.29.42.254:35087 requested a full tablet report, sending...
14:19:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.052740 29908 ts_manager.cc:194] Registered new tserver with Master: e869f4131f254720aaf95cdea256e4ea (127.29.42.194:33731)
14:19:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.158689 30148 catalog_manager.cc:1259] Loaded cluster ID: 413a7d26ba674a6681d54ef9ffd4eb1f
14:19:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.158864 30148 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a: loading cluster ID for follower catalog manager: success
14:19:32.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.162771 30148 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a: acquiring CA information for follower catalog manager: success
14:19:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.165299 30148 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c405bcd8ec2b4480b609bd30e32b598a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:19:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:32.372627 30422 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:32.372893 30422 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:32.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:32.372977 30422 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:32.380852 30422 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:32.381034 30422 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.42.195
14:19:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.390425 30422 tablet_server_runner.cc:78] Tablet server non-default flags:
14:19:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.42.212:43871
14:19:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/data
14:19:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/wal
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.42.195:0
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/data/info.pb
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.42.195
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.42.254:35087,127.29.42.253:38615,127.29.42.252:45293
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.42.195
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/logs
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:32.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.391753 30422 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.393462 30422 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:32.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:32.406340 30428 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:32.406917 30429 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:32.407426 30431 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:32.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.408010 30422 server_base.cc:1034] running on GCE node
14:19:32.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.408582 30422 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:32.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.409584 30422 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:32.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.410857 30422 hybrid_clock.cc:648] HybridClock initialized: now 1714659572410738 us; error 103 us; skew 500 ppm
14:19:32.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.411222 30422 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:32.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.414125 30422 webserver.cc:460] Webserver started at http://127.29.42.195:34155/ using document root <none> and password file <none>
14:19:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.414996 30422 fs_manager.cc:359] Metadata directory not provided
14:19:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.415110 30422 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.415380 30422 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.418262 30422 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/data/instance:
14:19:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e065174960554be69650a27b73b71300"
14:19:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:32 on dist-test-slave-q1bb"
14:19:32.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.419054 30422 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/wal/instance:
14:19:32.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e065174960554be69650a27b73b71300"
14:19:32.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:32 on dist-test-slave-q1bb"
14:19:32.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.424399 30422 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
14:19:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.427966 30437 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.429426 30422 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:19:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.429595 30422 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/wal
14:19:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e065174960554be69650a27b73b71300"
14:19:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:32 on dist-test-slave-q1bb"
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.429785 30422 fs_report.cc:383] FS layout report
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/wal
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/wal
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/data/data
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:32.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.455735 30422 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:32.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.456714 30422 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:32.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.457053 30422 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:32.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.458016 30422 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
14:19:32.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.459663 30422 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:19:32.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.459738 30422 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:19:32.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.459820 30422 ts_tablet_manager.cc:590] Registered 0 tablets
14:19:32.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.459856 30422 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:19:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.496335 30422 rpc_server.cc:307] RPC server started. Bound to: 127.29.42.195:45147
14:19:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.496377 30549 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.42.195:45147 every 8 connection(s)
14:19:32.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.497699 30422 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/data/info.pb
14:19:32.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.499966 29867 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30422
14:19:32.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.500200 29867 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster1999619201545718427/ts-2/wal/instance
14:19:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.521059 30552 heartbeater.cc:347] Connected to a master server at 127.29.42.253:38615
14:19:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.521445 30552 heartbeater.cc:464] Registering TS with master...
14:19:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.521440 30550 heartbeater.cc:347] Connected to a master server at 127.29.42.252:45293
14:19:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.521623 30550 heartbeater.cc:464] Registering TS with master...
14:19:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.521891 30551 heartbeater.cc:347] Connected to a master server at 127.29.42.254:35087
14:19:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.522029 30551 heartbeater.cc:464] Registering TS with master...
14:19:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.522596 30550 heartbeater.cc:511] Master 127.29.42.252:45293 requested a full tablet report, sending...
14:19:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.523309 30552 heartbeater.cc:511] Master 127.29.42.253:38615 requested a full tablet report, sending...
14:19:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.523984 30551 heartbeater.cc:511] Master 127.29.42.254:35087 requested a full tablet report, sending...
14:19:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.524363 30051 ts_manager.cc:194] Registered new tserver with Master: e065174960554be69650a27b73b71300 (127.29.42.195:45147)
14:19:32.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.525369 29980 ts_manager.cc:194] Registered new tserver with Master: e065174960554be69650a27b73b71300 (127.29.42.195:45147)
14:19:32.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.525382 30051 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.42.195:42029
14:19:32.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.525846 29908 ts_manager.cc:194] Registered new tserver with Master: e065174960554be69650a27b73b71300 (127.29.42.195:45147)
14:19:32.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.528849 29867 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
14:19:32.530 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:19:32.531 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:19:32.541 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.42.254"
port: 35087
}
bound_http_address {
host: "127.29.42.254"
port: 40783
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.42.253"
port: 38615
}
bound_http_address {
host: "127.29.42.253"
port: 38027
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.42.252"
port: 45293
}
bound_http_address {
host: "127.29.42.252"
port: 34571
}
}
}
14:19:32.547 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:19:32.550 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.29.42.193"
port: 45053
}
bound_http_address {
host: "127.29.42.193"
port: 34519
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.29.42.194"
port: 33731
}
bound_http_address {
host: "127.29.42.194"
port: 43143
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.29.42.195"
port: 45147
}
bound_http_address {
host: "127.29.42.195"
port: 34155
}
}
}
14:19:32.551 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:19:32.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:32.613524 30275 heartbeater.cc:503] Master 127.29.42.252:45293 was elected leader, sending a full tablet report...
14:19:32.705 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:19:32.818 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taske_AVaV/test-tmp
14:19:32.819 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
14:19:32.819 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taske_AVaV/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
14:19:32.821 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:19:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:33.050812 30415 heartbeater.cc:503] Master 127.29.42.252:45293 was elected leader, sending a full tablet report...
14:19:33.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:33.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:19:33.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:33.152734 30558 mini_oidc.cc:152] Starting JWKS server
14:19:33.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:33.155196 30558 webserver.cc:460] Webserver started at http://127.0.0.1:38285/ using document root <none> and password file <none>
14:19:33.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:33.155393 30558 mini_oidc.cc:189] Starting OIDC Discovery server
14:19:33.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:33.155890 30558 webserver.cc:460] Webserver started at http://127.0.0.1:35743/ using document root <none> and password file <none>
14:19:33.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:19:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/principal' for realm 'KRBTEST.COM',
14:19:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:19:33.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:33 dist-test-slave-q1bb krb5kdc[30569](info): setting up network...
14:19:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:19:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:33 dist-test-slave-q1bb krb5kdc[30569](info): set up 2 sockets
14:19:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:33 dist-test-slave-q1bb krb5kdc[30569](info): commencing operation
14:19:33.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:19:33.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:33.527892 30550 heartbeater.cc:503] Master 127.29.42.252:45293 was elected leader, sending a full tablet report...
14:19:34.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.089572 30558 mini_kdc.cc:121] Time spent starting KDC: real 0.925s user 0.009s sys 0.004s
14:19:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:19:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:19:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:19:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:19:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:19:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:19:34.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:19:34.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:19:34.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:19:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:19:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/test-user.keytab.
14:19:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/test-user.keytab.
14:19:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:34 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659574, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:19:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:19:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
14:19:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:19:34Z Disabled control of system clock
14:19:34.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.215.190@KRBTEST.COM; defaulting to no policy
14:19:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.215.190@KRBTEST.COM" created.
14:19:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.215.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.190.keytab.
14:19:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.215.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.190.keytab.
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.245803 30558 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/logs
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/info.pb
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.190:38579
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.190
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.190.keytab
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.190
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:19:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.562472 30582 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.562763 30582 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.562860 30582 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.570856 30582 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:19:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.570963 30582 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.571034 30582 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.571112 30582 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.571168 30582 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.571254 30582 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:34.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.571305 30582 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38285/jwks.json
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.581111 30582 master_runner.cc:386] Master server non-default flags:
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.190.keytab
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.190
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.190:38579
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/info.pb
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.190
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/logs
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.582422 30582 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.583547 30582 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:34.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.594460 30587 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.595050 30590 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.595716 30588 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:34.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.596365 30582 server_base.cc:1034] running on GCE node
14:19:34.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.597024 30582 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:34.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.598312 30582 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.599503 30582 hybrid_clock.cc:648] HybridClock initialized: now 1714659574599450 us; error 44 us; skew 500 ppm
14:19:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.599848 30582 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:34 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659574, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:34.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.602818 30582 init.cc:377] Logged in from keytab as kudu/127.29.215.190@KRBTEST.COM (short username kudu)
14:19:34.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.604105 30582 webserver.cc:460] Webserver started at http://127.29.215.190:45843/ using document root <none> and password file <none>
14:19:34.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.604918 30582 fs_manager.cc:359] Metadata directory not provided
14:19:34.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.605043 30582 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:34.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.605360 30582 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.607801 30582 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/instance:
14:19:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34ced298f4674c58bd1be4723fecbe44"
14:19:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:34 on dist-test-slave-q1bb"
14:19:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.608551 30582 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal/instance:
14:19:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34ced298f4674c58bd1be4723fecbe44"
14:19:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:34 on dist-test-slave-q1bb"
14:19:34.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.613487 30582 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
14:19:34.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.617085 30597 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.618330 30582 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
14:19:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.618500 30582 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34ced298f4674c58bd1be4723fecbe44"
14:19:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:34 on dist-test-slave-q1bb"
14:19:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.618695 30582 fs_report.cc:383] FS layout report
14:19:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/data
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:34.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.640697 30582 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.644891 30582 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.645109 30582 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.667171 30582 rpc_server.cc:307] RPC server started. Bound to: 127.29.215.190:38579
14:19:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.667197 30648 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.215.190:38579 every 8 connection(s)
14:19:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.668467 30582 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/info.pb
14:19:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.669987 30558 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30582
14:19:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.670233 30558 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal/instance
14:19:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.672142 30649 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.678258 30649 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } has no permanent_uuid. Determining permanent_uuid...
14:19:34.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.215.189@KRBTEST.COM; defaulting to no policy
14:19:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.215.189@KRBTEST.COM" created.
14:19:34.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:34 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659574, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.190@KRBTEST.COM for kudu/127.29.215.190@KRBTEST.COM
14:19:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.699559 30649 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } has no permanent_uuid. Determining permanent_uuid...
14:19:34.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.700776 30600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:34.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.702561 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.189:38973: Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.215.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.189.keytab.
14:19:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.215.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.189.keytab.
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.710278 30558 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/logs
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/info.pb
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.189:38973
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.189
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.189.keytab
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.189
14:19:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:34.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:19:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.751822 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } attempt: 1
14:19:34.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.753532 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.189:38973: Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:34.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.821868 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } attempt: 2
14:19:34.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.823839 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.189:38973: Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:34.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:34.915086 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } attempt: 3
14:19:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:34.916957 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.189:38973: Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.023785 30656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.024189 30656 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.024358 30656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:35.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.032364 30656 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:19:35.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.032450 30656 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:35.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.032498 30656 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:35.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.032544 30656 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:35.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.032579 30656 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:35.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.032634 30656 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:35.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.032663 30656 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38285/jwks.json
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.042274 30656 master_runner.cc:386] Master server non-default flags:
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.189.keytab
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.189
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.189:38973
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/info.pb
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.189
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/logs
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.043522 30656 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.044770 30656 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.054929 30661 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.055289 30662 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.055699 30664 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:35.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.056016 30656 server_base.cc:1034] running on GCE node
14:19:35.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.056777 30656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:35.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.058311 30656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:35.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.059509 30656 hybrid_clock.cc:648] HybridClock initialized: now 1714659575059460 us; error 36 us; skew 500 ppm
14:19:35.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.059846 30656 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:35.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.060168 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } attempt: 4
14:19:35.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.061635 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.189:38973: Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:35.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.068982 30656 init.cc:377] Logged in from keytab as kudu/127.29.215.189@KRBTEST.COM (short username kudu)
14:19:35.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.070626 30656 webserver.cc:460] Webserver started at http://127.29.215.189:45675/ using document root <none> and password file <none>
14:19:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.071296 30656 fs_manager.cc:359] Metadata directory not provided
14:19:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.071413 30656 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.071694 30656 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.074213 30656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/instance:
14:19:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78dea61621df469686088aa6527a80ab"
14:19:35.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:35 on dist-test-slave-q1bb"
14:19:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.074954 30656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal/instance:
14:19:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78dea61621df469686088aa6527a80ab"
14:19:35.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:35 on dist-test-slave-q1bb"
14:19:35.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.079690 30656 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
14:19:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.082741 30671 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.083931 30656 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
14:19:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.084100 30656 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78dea61621df469686088aa6527a80ab"
14:19:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:35 on dist-test-slave-q1bb"
14:19:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.084296 30656 fs_report.cc:383] FS layout report
14:19:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:35.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/data
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:35.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:35.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.122030 30656 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.126128 30656 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.126467 30656 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.144212 30656 rpc_server.cc:307] RPC server started. Bound to: 127.29.215.189:38973
14:19:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.144234 30722 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.215.189:38973 every 8 connection(s)
14:19:35.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.145512 30656 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/info.pb
14:19:35.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.149159 30723 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:35.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.153795 30558 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30656
14:19:35.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.153995 30558 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal/instance
14:19:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.154922 30723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } has no permanent_uuid. Determining permanent_uuid...
14:19:35.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.189@KRBTEST.COM for kudu/127.29.215.190@KRBTEST.COM
14:19:35.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.215.188@KRBTEST.COM; defaulting to no policy
14:19:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.215.188@KRBTEST.COM" created.
14:19:35.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.176671 30723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } has no permanent_uuid. Determining permanent_uuid...
14:19:35.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.189@KRBTEST.COM for kudu/127.29.215.189@KRBTEST.COM
14:19:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.191965 30723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } has no permanent_uuid. Determining permanent_uuid...
14:19:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.215.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.188.keytab.
14:19:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.215.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.188.keytab.
14:19:35.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.192857 30675 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.193907 30558 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/logs
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/info.pb
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.188:36453
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.188
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.188.keytab
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.188
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:19:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.195505 30723 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.244796 30723 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 1
14:19:35.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.246707 30723 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.314927 30723 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 2
14:19:35.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.316731 30723 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.360857 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } attempt: 5
14:19:35.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659574, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.190@KRBTEST.COM for kudu/127.29.215.189@KRBTEST.COM
14:19:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.375095 30649 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } has no permanent_uuid. Determining permanent_uuid...
14:19:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.376971 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.407936 30723 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 3
14:19:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.409952 30723 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.428238 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 1
14:19:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.429912 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.498113 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 2
14:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.499964 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.514353 30730 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:35.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.514642 30730 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:35.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.514791 30730 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:35.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.522109 30730 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:19:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.522205 30730 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.522255 30730 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.522307 30730 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.522367 30730 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.522444 30730 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.522488 30730 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38285/jwks.json
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.532253 30730 master_runner.cc:386] Master server non-default flags:
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.188.keytab
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.188
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.188:36453
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/info.pb
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.188
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/logs
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:35.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.533864 30730 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.535144 30730 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.545185 30735 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.545578 30738 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.546073 30736 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.546167 30730 server_base.cc:1034] running on GCE node
14:19:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.546872 30730 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:35.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.548205 30730 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:35.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.549394 30730 hybrid_clock.cc:648] HybridClock initialized: now 1714659575549331 us; error 57 us; skew 500 ppm
14:19:35.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.549688 30730 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:35.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:35.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.552744 30730 init.cc:377] Logged in from keytab as kudu/127.29.215.188@KRBTEST.COM (short username kudu)
14:19:35.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.553154 30723 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 4
14:19:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.554438 30730 webserver.cc:460] Webserver started at http://127.29.215.188:36843/ using document root <none> and password file <none>
14:19:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.554978 30723 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.555217 30730 fs_manager.cc:359] Metadata directory not provided
14:19:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.555316 30730 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.555552 30730 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:19:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.557994 30730 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/instance:
14:19:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84726e27507d4b4f85523cc71670008e"
14:19:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:35 on dist-test-slave-q1bb"
14:19:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.558641 30730 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal/instance:
14:19:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84726e27507d4b4f85523cc71670008e"
14:19:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:35 on dist-test-slave-q1bb"
14:19:35.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.563483 30730 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
14:19:35.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.566670 30745 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.567926 30730 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.568064 30730 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84726e27507d4b4f85523cc71670008e"
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:35 on dist-test-slave-q1bb"
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.568217 30730 fs_report.cc:383] FS layout report
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/data
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:35.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.589022 30730 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:35.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.593024 30730 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:35.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.593274 30730 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:35.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.606178 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 3
14:19:35.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.608229 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.610616 30730 rpc_server.cc:307] RPC server started. Bound to: 127.29.215.188:36453
14:19:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.610661 30796 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.215.188:36453 every 8 connection(s)
14:19:35.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.611860 30730 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/info.pb
14:19:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.615633 30797 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:35.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.617481 30558 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30730
14:19:35.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.617672 30558 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal/instance
14:19:35.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.624504 30797 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } has no permanent_uuid. Determining permanent_uuid...
14:19:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.188@KRBTEST.COM for kudu/127.29.215.190@KRBTEST.COM
14:19:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659574, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.215.188@KRBTEST.COM
14:19:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.642655 30797 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } has no permanent_uuid. Determining permanent_uuid...
14:19:35.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.188@KRBTEST.COM for kudu/127.29.215.189@KRBTEST.COM
14:19:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.655942 30797 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } has no permanent_uuid. Determining permanent_uuid...
14:19:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659574, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.215.190@KRBTEST.COM
14:19:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.188@KRBTEST.COM for kudu/127.29.215.188@KRBTEST.COM
14:19:35.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659574, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.215.189@KRBTEST.COM
14:19:35.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.677279 30797 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Bootstrap starting.
14:19:35.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.680442 30797 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Neither blocks nor log segments found. Creating new log.
14:19:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.681484 30797 log.cc:824] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Log is configured to *not* fsync() on all Append() calls
14:19:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.682184 30558 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
14:19:35.684 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:35.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.684783 30797 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: No bootstrap required, opened a new log
14:19:35.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:19:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.691370 30797 raft_consensus.cc:348] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.691641 30797 raft_consensus.cc:374] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:19:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.691721 30797 raft_consensus.cc:729] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84726e27507d4b4f85523cc71670008e, State: Initialized, Role: FOLLOWER
14:19:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.692271 30797 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.693204 30803 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } } }
14:19:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.693686 30803 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [sys.catalog]: This master's current role is: FOLLOWER
14:19:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.694681 30797 sys_catalog.cc:564] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [sys.catalog]: configured and running, proceeding with master startup.
14:19:35.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:19:35.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:19:35.705 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.215.190"
port: 38579
}
bound_http_address {
host: "127.29.215.190"
port: 45843
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.215.189"
port: 38973
}
bound_http_address {
host: "127.29.215.189"
port: 45675
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.215.188"
port: 36453
}
bound_http_address {
host: "127.29.215.188"
port: 36843
}
}
}
14:19:35.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:19:35.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:35.706 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:19:35.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:19:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.716706 30814 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.717031 30814 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.714329 30730 master_runner.cc:186] Error getting master registration for 127.29.215.190:38579: OK, instance_id { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" instance_seqno: 1714659574656126 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:19:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.723140 30730 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:35.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:35.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:19:35.736 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:35.741 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc
14:19:35.746 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:19:35.747 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc
14:19:35.748 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:19:35.752 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:19:35.753 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:19:35.754 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:19:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:35 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:35.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:19:35.779 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:35.779 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc
14:19:35.781 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:19:35.781 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.215.188:36453
14:19:35.783 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:19:35.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.783499 30558 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30730
14:19:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.785028 30792 generic_service.cc:196] Checking for leaks (request via RPC)
14:19:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.785462 30649 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 4
14:19:35.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.854192 30723 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } attempt: 5
14:19:35.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.880051 30751 tablet.cc:2369] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:19:35.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.881745 30558 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 30730
14:19:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.899346 30652 negotiation.cc:337] Failed RPC negotiation. Trace:
14:19:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.786139 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.29.215.188:36453 (local address 127.0.0.1:57198)
14:19:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.786369 (+ 230us) negotiation.cc:107] Waiting for socket to connect
14:19:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.786394 (+ 25us) client_negotiation.cc:174] Beginning negotiation
14:19:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.786552 (+ 158us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
14:19:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.881127 (+ 94575us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
14:19:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.881135 (+ 8us) client_negotiation.cc:394] Received NEGOTIATE response from server
14:19:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.881631 (+ 496us) client_negotiation.cc:189] Negotiated authn=SASL
14:19:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.882093 (+ 462us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
14:19:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.882109 (+ 16us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
14:19:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.898439 (+ 16330us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:19:35.900 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":52}
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.899638 30726 negotiation.cc:337] Failed RPC negotiation. Trace:
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.854779 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.29.215.188:36453 (local address 127.0.0.1:57212)
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.854944 (+ 165us) negotiation.cc:107] Waiting for socket to connect
14:19:35.901 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.215.189:38973
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.854959 (+ 15us) client_negotiation.cc:174] Beginning negotiation
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.855049 (+ 90us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.880354 (+ 25305us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.880362 (+ 8us) client_negotiation.cc:394] Received NEGOTIATE response from server
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.880938 (+ 576us) client_negotiation.cc:189] Negotiated authn=SASL
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.881435 (+ 497us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.881453 (+ 18us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0502 14:19:35.898440 (+ 16987us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: BlockingRecv error: recv got EOF from 127.29.215.188:36453 (error 108)
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":51}
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.900895 30649 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.901 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:35.901000 30723 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.29.215.188:36453: Network error: Client connection negotiation failed: client connection to 127.29.215.188:36453: connect: Connection refused (error 111)
14:19:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.901772 30558 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30656
14:19:35.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.902720 30718 generic_service.cc:196] Checking for leaks (request via RPC)
14:19:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.979545 30558 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 30656
14:19:35.996 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:35.996 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.215.190:38579
14:19:35.996 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:19:35.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.996564 30558 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30582
14:19:35.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:35.997754 30644 generic_service.cc:196] Checking for leaks (request via RPC)
14:19:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.138806 30558 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 30582
14:19:36.163 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:36.163 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.215.188:36453
14:19:36.164 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:19:36.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.165578 30558 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:36.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:36.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/logs
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/info.pb
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.188:36453
14:19:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.188
14:19:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36843
14:19:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.188.keytab
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.188
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:19:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.483086 30824 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.483403 30824 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.483506 30824 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.490592 30824 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:19:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.490700 30824 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.490749 30824 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.490798 30824 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.490834 30824 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.490883 30824 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.490914 30824 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38285/jwks.json
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.500119 30824 master_runner.cc:386] Master server non-default flags:
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.188.keytab
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.188
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.188:36453
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/info.pb
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.188
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36843
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/logs
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.501392 30824 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.502602 30824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:36.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.513175 30829 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:36.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.513231 30830 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:36.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.513751 30832 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:36.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.514379 30824 server_base.cc:1034] running on GCE node
14:19:36.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.515064 30824 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:36.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.516417 30824 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:36.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.517621 30824 hybrid_clock.cc:648] HybridClock initialized: now 1714659576517573 us; error 44 us; skew 500 ppm
14:19:36.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.517999 30824 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:36 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659576, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.521071 30824 init.cc:377] Logged in from keytab as kudu/127.29.215.188@KRBTEST.COM (short username kudu)
14:19:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.522670 30824 webserver.cc:460] Webserver started at http://127.29.215.188:36843/ using document root <none> and password file <none>
14:19:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.523406 30824 fs_manager.cc:359] Metadata directory not provided
14:19:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.523561 30824 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:36.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.528424 30824 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
14:19:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.531237 30839 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.532402 30824 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
14:19:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.532564 30824 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84726e27507d4b4f85523cc71670008e"
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:35 on dist-test-slave-q1bb"
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.533597 30824 fs_report.cc:383] FS layout report
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/wal
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/data
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:36.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.550015 30824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:36.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.553535 30824 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:36.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.553757 30824 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.570991 30824 rpc_server.cc:307] RPC server started. Bound to: 127.29.215.188:36453
14:19:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.570996 30890 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.215.188:36453 every 8 connection(s)
14:19:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.572746 30824 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-2/data/info.pb
14:19:36.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.579221 30891 sys_catalog.cc:263] Verifying existing consensus state
14:19:36.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.580468 30558 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30824
14:19:36.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:36.581 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.215.189:38973
14:19:36.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:19:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.581696 30558 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/logs
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/info.pb
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.189:38973
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.189
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45675
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.189.keytab
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.189
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:19:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:19:36.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.582695 30891 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Bootstrap starting.
14:19:36.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.595453 30891 log.cc:824] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Log is configured to *not* fsync() on all Append() calls
14:19:36.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.597514 30891 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:19:36.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.598043 30891 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Bootstrap complete.
14:19:36.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.606031 30891 raft_consensus.cc:348] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:36.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.606770 30891 raft_consensus.cc:374] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:19:36.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.606864 30891 raft_consensus.cc:729] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84726e27507d4b4f85523cc71670008e, State: Initialized, Role: FOLLOWER
14:19:36.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.607517 30891 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:36.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.608534 30895 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } } }
14:19:36.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.608875 30895 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [sys.catalog]: This master's current role is: FOLLOWER
14:19:36.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.609660 30891 sys_catalog.cc:564] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [sys.catalog]: configured and running, proceeding with master startup.
14:19:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.622479 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.622900 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:36.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.623605 30824 master_runner.cc:186] Error getting master registration for 127.29.215.190:38579: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111),
14:19:36.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.631222 30824 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:36.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.805931 30845 tablet.cc:2369] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:19:36.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.824447 30895 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:19:36.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.824717 30895 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:36.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.827330 30895 leader_election.cc:288] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579), 78dea61621df469686088aa6527a80ab (127.29.215.189:38973)
14:19:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.831954 30842 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:36.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.844842 30842 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 78dea61621df469686088aa6527a80ab (127.29.215.189:38973): Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:36.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.845448 30840 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579): Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111)
14:19:36.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.845762 30840 leader_election.cc:302] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84726e27507d4b4f85523cc71670008e; no voters: 34ced298f4674c58bd1be4723fecbe44, 78dea61621df469686088aa6527a80ab
14:19:36.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.846334 30895 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:19:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.919356 30893 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.919688 30893 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.919835 30893 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.927367 30893 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:19:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.927479 30893 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.927553 30893 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.927623 30893 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.927680 30893 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.927753 30893 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.927806 30893 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38285/jwks.json
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.937928 30893 master_runner.cc:386] Master server non-default flags:
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.189.keytab
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.189
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.189:38973
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/info.pb
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.189
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45675
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/logs
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.939615 30893 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:36.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.941026 30893 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:36.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.951887 30915 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:36.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.952435 30916 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:36.956413 30918 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:36.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.957350 30893 server_base.cc:1034] running on GCE node
14:19:36.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.958140 30893 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:36.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.959600 30893 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:36.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.960820 30893 hybrid_clock.cc:648] HybridClock initialized: now 1714659576960768 us; error 49 us; skew 500 ppm
14:19:36.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.961190 30893 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:36.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:36 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659576, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:36.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.964330 30893 init.cc:377] Logged in from keytab as kudu/127.29.215.189@KRBTEST.COM (short username kudu)
14:19:36.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.966087 30893 webserver.cc:460] Webserver started at http://127.29.215.189:45675/ using document root <none> and password file <none>
14:19:36.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.966768 30893 fs_manager.cc:359] Metadata directory not provided
14:19:36.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.966908 30893 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:36.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.972193 30893 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.003s
14:19:36.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.975026 30925 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:36.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.976415 30893 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.976596 30893 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78dea61621df469686088aa6527a80ab"
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:35 on dist-test-slave-q1bb"
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.977444 30893 fs_report.cc:383] FS layout report
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/data
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:36.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:36.998432 30893 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:37.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.002663 30893 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:37.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.002948 30893 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.020547 30893 rpc_server.cc:307] RPC server started. Bound to: 127.29.215.189:38973
14:19:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.020577 30976 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.215.189:38973 every 8 connection(s)
14:19:37.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.022109 30893 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/data/info.pb
14:19:37.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.027055 30558 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30893
14:19:37.028 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:37.028 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.215.190:38579
14:19:37.028 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.029155 30558 external_mini_cluster.cc:1326] Running /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske_AVaV/build/asan/bin/kudu
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/logs
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/info.pb
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.190:38579
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.190
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45843
14:19:37.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.190.keytab
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.190
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.029685 30977 sys_catalog.cc:263] Verifying existing consensus state
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 14:19:37.030145 30977 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:19:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.030354 30893 master.cc:537] Master@127.29.215.189:38973 shutting down...
14:19:37.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.054100 30893 master.cc:552] Master@127.29.215.189:38973 shutdown complete.
14:19:37.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:19:37.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.367491 30979 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.367800 30979 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.367930 30979 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.376219 30979 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:19:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.376324 30979 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.376375 30979 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.376425 30979 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.376462 30979 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.376511 30979 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.376543 30979 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38285/jwks.json
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.386241 30979 master_runner.cc:386] Master server non-default flags:
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.215.148:33283
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.215.190:38579,127.29.215.189:38973,127.29.215.188:36453
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/krb5kdc/kudu_127.29.215.190.keytab
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.215.190
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.215.190:38579
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38285/jwks.json
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:19:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:19:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/info.pb
14:19:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.215.190
14:19:37.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45843
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/logs
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 91
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.387524 30979 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.388772 30979 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:37.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.398955 30986 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.399668 30985 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.400342 30979 server_base.cc:1034] running on GCE node
14:19:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.401110 30988 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:37.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.402350 30979 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.404152 30979 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.405319 30979 hybrid_clock.cc:648] HybridClock initialized: now 1714659577405241 us; error 64 us; skew 500 ppm
14:19:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.405762 30979 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:37.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:37 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659577, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:37.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.410198 30979 init.cc:377] Logged in from keytab as kudu/127.29.215.190@KRBTEST.COM (short username kudu)
14:19:37.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.413990 30979 webserver.cc:460] Webserver started at http://127.29.215.190:45843/ using document root <none> and password file <none>
14:19:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.414904 30979 fs_manager.cc:359] Metadata directory not provided
14:19:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.415048 30979 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:37.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.421454 30979 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.005s sys 0.001s
14:19:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.424932 30995 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.426276 30979 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.426411 30979 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data,/tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34ced298f4674c58bd1be4723fecbe44"
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:19:34 on dist-test-slave-q1bb"
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.427179 30979 fs_report.cc:383] FS layout report
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/data
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.459563 30979 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.463174 30979 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.463376 30979 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:37.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.479774 30979 rpc_server.cc:307] RPC server started. Bound to: 127.29.215.190:38579
14:19:37.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.479816 31046 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.215.190:38579 every 8 connection(s)
14:19:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.481406 30979 server_base.cc:1166] Dumped server information to /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/data/info.pb
14:19:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.487035 31047 sys_catalog.cc:263] Verifying existing consensus state
14:19:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 14:19:37.487466 31047 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:19:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.487691 30979 master.cc:537] Master@127.29.215.190:38579 shutting down...
14:19:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.487898 30558 external_mini_cluster.cc:1388] Started /tmp/dist-test-taske_AVaV/build/asan/bin/kudu as pid 30979
14:19:37.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:37.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:37.510969 30979 master.cc:552] Master@127.29.215.190:38579 shutdown complete.
14:19:37.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taske_AVaV/test-tmp/mini-kudu-cluster4939504454280696006/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:19:37.537 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.215.188:36453(127.29.215.188:36453)] connecting to peer
14:19:37.598 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:37.606 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.625039 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:37.625207 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:37.650 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.215.188:36453(127.29.215.188:36453)] Successfully connected to peer
14:19:37.650 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.700 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.716 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:19:37.723 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.29.215.188
14:19:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:37 dist-test-slave-q1bb krb5kdc[30569](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659575, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.215.188@KRBTEST.COM
14:19:37.921 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe00bcda4, L:/127.0.0.1:60810 - R:/127.29.215.188:36453] using SASL/GSSAPI
14:19:37.938 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:37.956 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:37.958 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:37.959 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.960 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.961 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.963 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.963 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.964 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.964 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:37.973 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:37.974 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:37.976 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.976 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.977 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.979 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:37.993 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:37.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:37.996 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.996 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.997 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:37.997 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:37.998 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:38.013 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:38.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:38.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.018 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.018 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:38.053 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:38.054 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:38.056 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.056 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.057 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.058 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:38.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:38.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:38.116 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.116 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.116 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.119 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:38.193 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:38.196 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:38.197 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.197 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.197 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.199 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:38.333 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:38.334 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:38.336 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.336 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.337 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.337 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.338 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:38.433 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:38.435 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:38.436 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.436 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.437 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.438 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.438 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.438 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.439 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:38.473 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:38.474 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:38.476 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.476 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.476 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:38.476 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.477 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.477 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:38.477 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:38.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:38.626441 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:38.626551 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:38.851801 31054 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:19:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:38.851996 31054 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:38.852633 31054 leader_election.cc:288] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579), 78dea61621df469686088aa6527a80ab (127.29.215.189:38973)
14:19:38.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:38.854705 30842 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 78dea61621df469686088aa6527a80ab (127.29.215.189:38973): Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:38.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:38.854889 30840 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579): Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111)
14:19:38.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:38.855006 30840 leader_election.cc:302] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84726e27507d4b4f85523cc71670008e; no voters: 34ced298f4674c58bd1be4723fecbe44, 78dea61621df469686088aa6527a80ab
14:19:38.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:38.855222 31054 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:19:39.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:39.627642 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:39.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:39.627758 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:40.273 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:40.274 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:40.275 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:40.276 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:40.276 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:40.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:40.277 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:40.277 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:40.277 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:40.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:40.420845 31058 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:19:40.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:40.421057 31058 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:40.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:40.421921 31058 leader_election.cc:288] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579), 78dea61621df469686088aa6527a80ab (127.29.215.189:38973)
14:19:40.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:40.423981 30840 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579): Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111)
14:19:40.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:40.424161 30842 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 78dea61621df469686088aa6527a80ab (127.29.215.189:38973): Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:40.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:40.424312 30842 leader_election.cc:302] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84726e27507d4b4f85523cc71670008e; no voters: 34ced298f4674c58bd1be4723fecbe44, 78dea61621df469686088aa6527a80ab
14:19:40.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:40.424566 31058 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:19:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:40.628858 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:40.628973 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:41.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:41.630079 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:41.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:41.630203 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:41.632594 30824 master_runner.cc:186] Error getting master registration for 127.29.215.190:38579: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111),
14:19:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:41.632772 30824 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:41.673 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:41.675 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:41.675 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:41.675 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:41.676 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:41.676 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:41.677 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:41.677 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:41.677 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:42.631243 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:42.631392 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:43.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:43.632532 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:43.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:43.632663 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:43.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:43.728652 31064 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:19:43.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:43.728860 31064 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:43.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:43.729617 31064 leader_election.cc:288] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579), 78dea61621df469686088aa6527a80ab (127.29.215.189:38973)
14:19:43.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:43.730562 30840 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111) [suppressed 5 similar messages]
14:19:43.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:43.731710 30842 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 78dea61621df469686088aa6527a80ab (127.29.215.189:38973): Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:43.731962 30840 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579): Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111)
14:19:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:43.732102 30840 leader_election.cc:302] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84726e27507d4b4f85523cc71670008e; no voters: 34ced298f4674c58bd1be4723fecbe44, 78dea61621df469686088aa6527a80ab
14:19:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:43.732374 31064 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:19:44.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:44.634197 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:44.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:44.634330 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:44.674 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:44.675 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:44.676 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:44.676 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:44.676 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:44.677 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:44.677 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:44.677 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:44.677 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:45.593 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:45.595 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:45.596 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:45.597 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:45.597 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:45.597 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:45.597 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:45.597 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:45.597 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:45.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:45.635733 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:45.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:45.635849 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:46.373 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:46.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:46.376 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:46.376 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:46.376 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:46.377 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:46.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:46.377 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:46.377 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:46.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:46 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659586, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:46.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:46.523224 30835 init.cc:295] Successfully reacquired a new kerberos TGT
14:19:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:46.634423 30824 master_runner.cc:186] Error getting master registration for 127.29.215.190:38579: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111),
14:19:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:46.634639 30824 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:46.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:46.637169 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:46.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:46.637295 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:46.993 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:46.996 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:46.996 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:46.996 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:46.997 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:46.997 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:46.997 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:46.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:46.998 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:47.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:47.638682 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:47.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:47.638808 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:47.993 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:47.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:47.995 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:47.995 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:47.996 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:47.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:47.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:47.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:47.999 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:48.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:48.640221 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:48.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:48.640370 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:49.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:49.641709 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:49.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:49.641855 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:49.840788 31070 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:19:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:49.841010 31070 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:49.841856 31070 leader_election.cc:288] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579), 78dea61621df469686088aa6527a80ab (127.29.215.189:38973)
14:19:49.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:49.843037 30840 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:19:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:49.844415 30840 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579): Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111)
14:19:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:49.844600 30842 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 78dea61621df469686088aa6527a80ab (127.29.215.189:38973): Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:49.844741 30842 leader_election.cc:302] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84726e27507d4b4f85523cc71670008e; no voters: 34ced298f4674c58bd1be4723fecbe44, 78dea61621df469686088aa6527a80ab
14:19:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:49.845077 31070 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:19:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:50.643142 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:50.643317 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:50.833 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:50.834 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:50.835 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:50.835 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:50.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:50.837 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:50.837 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:50.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:50.837 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:51.636077 30824 master_runner.cc:186] Error getting master registration for 127.29.215.190:38579: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111),
14:19:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:51.636255 30824 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:51.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:51.644420 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:51.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:51.644552 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:52.493 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:52.496 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:52.496 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:52.496 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:52.496 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:52.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:52.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:52.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:52.498 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:52.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:52.645769 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:52.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:52.645936 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:53.233 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:53.233 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:53.234 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:53.234 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:53.234 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:53.234 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:53.234 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:53.235 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:53.235 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:53.647274 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:53.647413 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:54.648628 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:54.648746 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:55.649960 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:55.650087 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:55.978142 31076 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:19:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:55.978400 31076 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:19:55.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:55.979203 31076 leader_election.cc:288] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579), 78dea61621df469686088aa6527a80ab (127.29.215.189:38973)
14:19:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:55.980175 30842 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:19:55.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:55.981537 30842 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 78dea61621df469686088aa6527a80ab (127.29.215.189:38973): Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:19:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:55.981992 30840 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579): Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111)
14:19:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:55.982141 30840 leader_election.cc:302] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84726e27507d4b4f85523cc71670008e; no voters: 34ced298f4674c58bd1be4723fecbe44, 78dea61621df469686088aa6527a80ab
14:19:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:55.982460 31076 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:19:56.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:19:56 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659596, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:56.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:56.524791 30835 init.cc:295] Successfully reacquired a new kerberos TGT
14:19:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:56.643640 30824 master_runner.cc:186] Error getting master registration for 127.29.215.190:38579: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111),
14:19:56.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:19:56.643813 30824 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:56.651252 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:56.651343 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:56.953 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:56.954 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:56.954 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:56.954 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:56.954 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:56.954 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:56.955 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:56.955 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:56.955 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:57.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:57.652599 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:57.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:57.652752 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:58.653929 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:58.654048 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:58.913 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:58.914 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:58.914 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:58.914 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:58.915 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:58.915 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:58.915 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:58.915 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:58.916 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:19:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:59.655133 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:19:59.655261 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:59.934 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:19:59.934 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:19:59.935 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:59.935 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:59.935 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:19:59.936 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:59.936 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:59.937 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:19:59.937 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:20:00.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:00.656446 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:00.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:00.656575 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:01.333 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:20:01.334 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:20:01.334 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:01.335 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:01.335 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:01.335 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:01.335 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:01.335 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:01.336 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:20:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:01.645315 30824 master_runner.cc:186] Error getting master registration for 127.29.215.190:38579: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111),
14:20:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:01.645408 31083 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:01.645573 30824 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:01.645618 31083 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34ced298f4674c58bd1be4723fecbe44" member_type: VOTER last_known_addr { host: "127.29.215.190" port: 38579 } } peers { permanent_uuid: "78dea61621df469686088aa6527a80ab" member_type: VOTER last_known_addr { host: "127.29.215.189" port: 38973 } } peers { permanent_uuid: "84726e27507d4b4f85523cc71670008e" member_type: VOTER last_known_addr { host: "127.29.215.188" port: 36453 } }
14:20:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:01.646322 31083 leader_election.cc:288] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579), 78dea61621df469686088aa6527a80ab (127.29.215.189:38973)
14:20:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:01.646765 30840 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:20:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:01.648039 30840 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 34ced298f4674c58bd1be4723fecbe44 (127.29.215.190:38579): Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111)
14:20:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:01.648239 30842 leader_election.cc:334] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 78dea61621df469686088aa6527a80ab (127.29.215.189:38973): Network error: Client connection negotiation failed: client connection to 127.29.215.189:38973: connect: Connection refused (error 111)
14:20:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:01.648394 30842 leader_election.cc:302] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 84726e27507d4b4f85523cc71670008e; no voters: 34ced298f4674c58bd1be4723fecbe44, 78dea61621df469686088aa6527a80ab
14:20:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:01.648666 31083 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:20:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:01.657702 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:01.657838 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:02.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:02.658977 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:02.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:02.659097 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:03.373 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:20:03.373 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:20:03.374 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:03.374 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:03.374 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:03.374 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:03.374 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:03.375 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:03.376 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:20:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:03.660267 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:03.660398 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:04.661526 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:04.661669 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:05.662912 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:05.663038 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:06.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:20:06 dist-test-slave-q1bb krb5kdc[30569](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659606, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.215.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:06.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:06.526419 30835 init.cc:295] Successfully reacquired a new kerberos TGT
14:20:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:06.647047 30824 master_runner.cc:186] Error getting master registration for 127.29.215.190:38579: Network error: Client connection negotiation failed: client connection to 127.29.215.190:38579: connect: Connection refused (error 111),
14:20:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:06.647244 30824 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:06.664131 30906 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 84726e27507d4b4f85523cc71670008e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:06.664266 30906 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:06.873 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.189:38973(127.29.215.189:38973)] connecting to peer
14:20:06.874 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.29.215.190:38579(127.29.215.190:38579)] connecting to peer
14:20:06.874 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:06.875 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:06.875 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.215.189:38973(127.29.215.189:38973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:06.875 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.190:38579(127.29.215.190:38579)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:06.875 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.190:38579: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579
14:20:06.875 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.215.189:38973: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
14:20:06.876 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579; will retry
14:20:06.879 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29384), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [157ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [204ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [440ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [443ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [458ms] refreshing cache from master, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [463ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [465ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [466ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [468ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [470ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [477ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [483ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [484ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [503ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [504ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [523ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [524ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [564ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [618ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [624ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [697ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [704ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [705ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [838ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [841ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [844ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [944ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [982ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2777ms] refreshing cache from master, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [2779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [2782ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2782ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [4177ms] refreshing cache from master, [4177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [4178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [4180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [4180ms] too many traces: truncated at 100 traces, deferred=Deferred@43421146(state=PENDING, result=null, callback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931), errback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931)))
14:20:06.882 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.215.189:38973(127.29.215.189:38973)] disconnecting while in state TERMINATED
14:20:06.883 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.215.188:36453(127.29.215.188:36453)] disconnecting while in state READY
14:20:06.883 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.215.190:38579(127.29.215.190:38579)] disconnecting while in state TERMINATED
14:20:06.887 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:20:06.888 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:20:06.890 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.215.188:36453(127.29.215.188:36453)] handling channelInactive
14:20:06.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.215.188:36453(127.29.215.188:36453)] cleaning up while in state READY due to: connection closed
14:20:06.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:06.893643 30558 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30824
14:20:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:06.900590 30558 external_mini_cluster.cc:1652] Unable to check leaks on /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30824: Not authorized: Client connection negotiation failed: client connection to 127.29.215.188:36453: server requires authentication, but client does not have Kerberos credentials available
14:20:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:06.900810 30558 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 30824
14:20:06.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:20:06Z chronyd exiting
14:20:07.045 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:20:07.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.046428 29867 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30130
14:20:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.059935 30270 generic_service.cc:196] Checking for leaks (request via RPC)
14:20:07.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.145725 29867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 30130
14:20:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.163895 29867 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30283
14:20:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.173868 30410 generic_service.cc:196] Checking for leaks (request via RPC)
14:20:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.264240 29867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 30283
14:20:07.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.287245 29867 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30422
14:20:07.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.295233 30545 generic_service.cc:196] Checking for leaks (request via RPC)
14:20:07.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.379983 29867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 30422
14:20:07.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.398001 29867 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 29878
14:20:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.399122 29939 generic_service.cc:196] Checking for leaks (request via RPC)
14:20:07.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.514951 29867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 29878
14:20:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:07.536389 30036 connection.cc:535] client connection to 127.29.42.254:35087 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:20:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:07.536652 30036 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:20:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.536732 29867 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 29946
14:20:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.537775 30010 generic_service.cc:196] Checking for leaks (request via RPC)
14:20:07.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:20:07.538626 30036 consensus_peers.cc:485] T 00000000000000000000000000000000 P 5766b19f4964424aae4d9474984e1b79 -> Peer d1333a1b2d984588a283cad84b4a7b46 (127.29.42.254:35087): Couldn't send request to peer d1333a1b2d984588a283cad84b4a7b46. Status: Network error: Client connection negotiation failed: client connection to 127.29.42.254:35087: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:20:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.639178 29867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 29946
14:20:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.660295 29867 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taske_AVaV/build/asan/bin/kudu pid 30017
14:20:07.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.661398 30081 generic_service.cc:196] Checking for leaks (request via RPC)
14:20:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:20:07.763136 29867 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taske_AVaV/build/asan/bin/kudu with pid 30017
14:20:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:20:07Z chronyd exiting
14:20:07.909 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29384), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [157ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [204ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [440ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [443ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [458ms] refreshing cache from master, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [463ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [465ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [466ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [468ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [470ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [477ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [483ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [484ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [503ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [504ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [523ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [524ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [564ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [618ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [624ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [697ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [704ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [705ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [838ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [841ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [844ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [944ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [982ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2777ms] refreshing cache from master, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [2779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [2782ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2782ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [4177ms] refreshing cache from master, [4177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [4178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [4180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [4180ms] too many traces: truncated at 100 traces, deferred=Deferred@43421146(state=PENDING, result=null, callback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931), errback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.215.189:38973
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29384), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [157ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [204ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [440ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [443ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [458ms] refreshing cache from master, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [463ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [465ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [466ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [468ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [470ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [477ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [483ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [484ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [503ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [504ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [523ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [524ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [564ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [618ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [624ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [697ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [704ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [705ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [838ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [841ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [844ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [944ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [982ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2777ms] refreshing cache from master, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [2779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [2782ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2782ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [4177ms] refreshing cache from master, [4177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [4178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [4180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [4180ms] too many traces: truncated at 100 traces, deferred=Deferred@43421146(state=PENDING, result=null, callback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931), errback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.215.189:38973
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14:20:08.665 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
14:20:08.665 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29384), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [157ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [204ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [440ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [443ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [458ms] refreshing cache from master, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [463ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [465ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [466ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [468ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [470ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [477ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [483ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [484ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [503ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [504ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [523ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [524ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [564ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [618ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [624ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [697ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [704ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [705ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [838ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [841ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [844ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [944ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [982ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2777ms] refreshing cache from master, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [2779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [2782ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2782ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [4177ms] refreshing cache from master, [4177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [4178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [4180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [4180ms] too many traces: truncated at 100 traces, deferred=Deferred@43421146(state=PENDING, result=null, callback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931), errback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.215.189:38973
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29384), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [157ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [204ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [440ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [443ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [458ms] refreshing cache from master, [458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [463ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [465ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [466ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [468ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [470ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [477ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [482ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [483ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [484ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [498ms] refreshing cache from master, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [501ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [503ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [504ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [521ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [523ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [524ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [562ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [564ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [618ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [621ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [624ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [697ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [704ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [705ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [838ms] refreshing cache from master, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [841ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [842ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [844ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [942ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [944ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [981ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [982ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2777ms] refreshing cache from master, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [2778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [2779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.189:38973: Network error: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973, [2781ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.188:36453: OK, [2782ms] Sub RPC ConnectToMaster: received response from server master-127.29.215.190:38579: Network error: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [2782ms] delaying RPC due to: Service unavailable: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579, [4177ms] refreshing cache from master, [4177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.188:36453, [4178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.189:38973, [4180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.215.190:38579, [4180ms] too many traces: truncated at 100 traces, deferred=Deferred@43421146(state=PENDING, result=null, callback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931), errback=(continuation of Deferred@402838328 after retry RPC after error@1319710566) -> (continuation of Deferred@703393469 after retry RPC after error@522345991) -> (continuation of Deferred@297256392 after retry RPC after error@375528071) -> (continuation of Deferred@1479907312 after retry RPC after error@1433020567) -> (continuation of Deferred@2061250167 after retry RPC after error@2095883657) -> (continuation of Deferred@143800307 after retry RPC after error@205035750) -> (continuation of Deferred@395092096 after retry RPC after error@240488213) -> (continuation of Deferred@392236163 after retry RPC after error@1599494433) -> (continuation of Deferred@766274283 after retry RPC after error@253982060) -> (continuation of Deferred@272196895 after retry RPC after error@2110499176) -> (continuation of Deferred@872312584 after retry RPC after error@660792334) -> (continuation of Deferred@2091416672 after retry RPC after error@1544034932) -> (continuation of Deferred@483309778 after retry RPC after error@742157777) -> (continuation of Deferred@323505994 after retry RPC after error@261045485) -> (continuation of Deferred@1726877158 after retry RPC after error@703833615) -> (continuation of Deferred@2014355044 after retry RPC after error@244231647) -> (continuation of Deferred@412680405 after retry RPC after error@745414167) -> (continuation of Deferred@1349600832 after retry RPC after error@728426948) -> (continuation of Deferred@2066078676 after retry RPC after error@669514753) -> (continuation of Deferred@2113859067 after retry RPC after error@27664201) -> (continuation of Deferred@1418235858 after retry RPC after error@1276606134) -> (continuation of Deferred@1183632378 after retry RPC after error@2002901045) -> (continuation of Deferred@1736916756 after retry RPC after error@273831828) -> (continuation of Deferred@807509375 after retry RPC after error@1241013054) -> (continuation of Deferred@400138890 after retry RPC after error@364226834) -> (continuation of Deferred@700646590 after retry RPC after error@585734931)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.215.188:36453,127.29.215.189:38973,127.29.215.190:38579) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.190:38579(127.29.215.190:38579): Connection refused: /127.29.215.190:38579,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.215.189:38973(127.29.215.189:38973): Connection refused: /127.29.215.189:38973
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.215.189:38973
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)