14:12:52.929 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:12:52.943 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQ4kZtv/test-tmp
14:12:52.945 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:12:52.948 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQ4kZtv/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:12:53.164 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:12:53.335 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:12:53.336 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:12:53.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.341758 21514 mini_oidc.cc:152] Starting JWKS server
14:12:53.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.345947 21514 webserver.cc:460] Webserver started at http://127.0.0.1:35213/ using document root <none> and password file <none>
14:12:53.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.346345 21514 mini_oidc.cc:189] Starting OIDC Discovery server
14:12:53.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.348403 21514 webserver.cc:460] Webserver started at http://127.0.0.1:35135/ using document root <none> and password file <none>
14:12:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:12:53Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
14:12:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:12:53Z Disabled control of system clock
14:12:53.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.408720 21514 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:12:53.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:12:53.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/wal
14:12:53.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/data
14:12:53.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:12:53.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:12:53.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:12:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:12:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:12:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:12:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/logs
14:12:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/data/info.pb
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:12:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:12:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:12:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:12:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:12:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.190:46093
14:12:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.190
14:12:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:12:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:12:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:12:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:12:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:12:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:12:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:12:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:12:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35213/jwks.json with env {}
14:12:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.700031 21526 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:12:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.700578 21526 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:12:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.701326 21526 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:12:53.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.731453 21526 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:12:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.731745 21526 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:12:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.731940 21526 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:12:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.732187 21526 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:12:53.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.732473 21526 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:12:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.732739 21526 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35213/jwks.json
14:12:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.766558 21526 master_runner.cc:386] Master server non-default flags:
14:12:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:12:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:12:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:12:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:12:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/data
14:12:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/wal
14:12:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:12:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:12:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:12:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:12:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:12:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.190:46093
14:12:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35213/jwks.json
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/data/info.pb
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.190
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:12:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/logs
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.767881 21526 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:12:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.769919 21526 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:12:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.786885 21535 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.787206 21533 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:53.787879 21532 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:53.788379 21526 server_base.cc:1034] running on GCE node
14:12:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.860352 21526 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:12:54.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.863013 21526 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:12:54.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.864384 21526 hybrid_clock.cc:648] HybridClock initialized: now 1714659174864352 us; error 47 us; skew 500 ppm
14:12:54.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.865231 21526 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:12:54.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.872079 21526 webserver.cc:460] Webserver started at http://127.21.2.190:45393/ using document root <none> and password file <none>
14:12:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.873023 21526 fs_manager.cc:359] Metadata directory not provided
14:12:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.873215 21526 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:12:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.873606 21526 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:12:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.878619 21526 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/data/instance:
14:12:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02b16b44ded54d9c892cdd338ea6149e"
14:12:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:54 on dist-test-slave-smf8"
14:12:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.879794 21526 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/wal/instance:
14:12:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02b16b44ded54d9c892cdd338ea6149e"
14:12:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:54 on dist-test-slave-smf8"
14:12:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.886906 21526 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
14:12:54.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.892761 21542 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:12:54.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.893874 21526 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:12:54.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.894181 21526 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/wal
14:12:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "02b16b44ded54d9c892cdd338ea6149e"
14:12:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:54 on dist-test-slave-smf8"
14:12:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.894483 21526 fs_report.cc:383] FS layout report
14:12:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:12:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/wal
14:12:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/wal
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/data/data
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:12:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:12:54.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:12:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.947577 21526 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:12:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.949071 21526 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:12:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:54.949493 21526 kserver.cc:163] Server-wide thread pool size limit: 3276
14:12:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.024600 21526 rpc_server.cc:307] RPC server started. Bound to: 127.21.2.190:46093
14:12:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.024724 21593 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.2.190:46093 every 8 connection(s)
14:12:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.027215 21526 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/data/info.pb
14:12:55.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.031638 21514 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 21526
14:12:55.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.032171 21514 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-0/wal/instance
14:12:55.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.033188 21594 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:12:55.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.037537 21514 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/wal
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/data
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/logs
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/data/info.pb
14:12:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:12:55.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.189:33785
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.189
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:12:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:12:55.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35213/jwks.json with env {}
14:12:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.048210 21594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } has no permanent_uuid. Determining permanent_uuid...
14:12:55.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.074430 21594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } has no permanent_uuid. Determining permanent_uuid...
14:12:55.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.076711 21543 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.2.189:33785: connect: Connection refused (error 111)
14:12:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.080953 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.189:33785: Network error: Client connection negotiation failed: client connection to 127.21.2.189:33785: connect: Connection refused (error 111)
14:12:55.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.130378 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } attempt: 1
14:12:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.135063 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.189:33785: Network error: Client connection negotiation failed: client connection to 127.21.2.189:33785: connect: Connection refused (error 111)
14:12:55.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.203438 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } attempt: 2
14:12:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.207093 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.189:33785: Network error: Client connection negotiation failed: client connection to 127.21.2.189:33785: connect: Connection refused (error 111)
14:12:55.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.298593 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } attempt: 3
14:12:55.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.302538 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.189:33785: Network error: Client connection negotiation failed: client connection to 127.21.2.189:33785: connect: Connection refused (error 111)
14:12:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.343559 21596 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:12:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.344141 21596 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:12:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.344756 21596 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:12:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.374845 21596 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:12:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.375147 21596 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:12:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.375339 21596 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:12:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.375515 21596 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:12:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.375686 21596 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:12:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.375823 21596 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35213/jwks.json
14:12:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.408748 21596 master_runner.cc:386] Master server non-default flags:
14:12:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:12:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:12:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:12:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:12:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/data
14:12:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/wal
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.189:33785
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35213/jwks.json
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:12:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/data/info.pb
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.189
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/logs
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:12:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:12:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:12:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:12:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:12:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:12:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:12:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:12:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.410001 21596 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:12:55.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.411594 21596 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:12:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.426318 21605 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.445978 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } attempt: 4
14:12:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.451033 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.189:33785: Network error: Client connection negotiation failed: client connection to 127.21.2.189:33785: connect: Connection refused (error 111)
14:12:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.750435 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } attempt: 5
14:12:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.755545 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.189:33785: Network error: Client connection negotiation failed: client connection to 127.21.2.189:33785: connect: Connection refused (error 111)
14:12:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.302933 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } attempt: 6
14:12:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:56.312889 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.189:33785: Network error: Client connection negotiation failed: client connection to 127.21.2.189:33785: connect: Connection refused (error 111)
14:12:56.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.426928 21608 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:56.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:55.427965 21606 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:56.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:55.428544 21596 server_base.cc:1034] running on GCE node
14:12:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.520910 21596 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:12:56.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.524252 21596 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:12:56.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.525712 21596 hybrid_clock.cc:648] HybridClock initialized: now 1714659176525674 us; error 57 us; skew 500 ppm
14:12:56.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.526758 21596 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:12:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.538053 21596 webserver.cc:460] Webserver started at http://127.21.2.189:35205/ using document root <none> and password file <none>
14:12:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.538998 21596 fs_manager.cc:359] Metadata directory not provided
14:12:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.539188 21596 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:12:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.539589 21596 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:12:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.544499 21596 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/data/instance:
14:12:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5fda3b8cbfb415c8444b4db8bad6284"
14:12:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:56 on dist-test-slave-smf8"
14:12:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.545686 21596 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/wal/instance:
14:12:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5fda3b8cbfb415c8444b4db8bad6284"
14:12:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:56 on dist-test-slave-smf8"
14:12:56.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.552949 21596 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
14:12:56.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.558439 21617 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:12:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.559526 21596 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
14:12:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.559881 21596 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/wal
14:12:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5fda3b8cbfb415c8444b4db8bad6284"
14:12:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:56 on dist-test-slave-smf8"
14:12:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.560261 21596 fs_report.cc:383] FS layout report
14:12:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:12:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/wal
14:12:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/wal
14:12:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/data/data
14:12:56.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:12:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:12:56.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.624048 21596 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:12:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.625675 21596 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:12:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.626178 21596 kserver.cc:163] Server-wide thread pool size limit: 3276
14:12:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.693368 21596 rpc_server.cc:307] RPC server started. Bound to: 127.21.2.189:33785
14:12:56.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.693455 21668 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.2.189:33785 every 8 connection(s)
14:12:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.696230 21596 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/data/info.pb
14:12:56.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.700354 21514 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 21596
14:12:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.700755 21514 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-1/wal/instance
14:12:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.701586 21669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.703720 21514 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/wal
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/data
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/logs
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/data/info.pb
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.188:36993
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.188
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:12:56.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:12:56.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:12:56.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:12:56.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:12:56.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:12:56.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:12:56.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35213/jwks.json with env {}
14:12:56.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.718102 21669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } has no permanent_uuid. Determining permanent_uuid...
14:12:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.743554 21669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } has no permanent_uuid. Determining permanent_uuid...
14:12:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.757234 21669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } has no permanent_uuid. Determining permanent_uuid...
14:12:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:56.759615 21619 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:56.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:56.763973 21669 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.813470 21669 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 1
14:12:56.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:56.817308 21669 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:56.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.885763 21669 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 2
14:12:56.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:56.889618 21669 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:56.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:56.981019 21669 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 3
14:12:56.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:56.986241 21669 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:57.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.017388 21671 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:12:57.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.018252 21671 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:12:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.019508 21671 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:12:57.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.051807 21671 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:12:57.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.052080 21671 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:12:57.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.052255 21671 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:12:57.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.052461 21671 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:12:57.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.052695 21671 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:12:57.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.052827 21671 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35213/jwks.json
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.086647 21671 master_runner.cc:386] Master server non-default flags:
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/data
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/wal
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.188:36993
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35213/jwks.json
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:12:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/data/info.pb
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.188
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/logs
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:12:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:12:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:12:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:12:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:12:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.087808 21671 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:12:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.089712 21671 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:12:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.107627 21682 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.129604 21669 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 4
14:12:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.134810 21669 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:57.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.373390 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } attempt: 7
14:12:57.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.384874 21594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } has no permanent_uuid. Determining permanent_uuid...
14:12:57.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.391155 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:57.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.434231 21669 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 5
14:12:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.438637 21669 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.449839 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 1
14:12:57.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.453953 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:57.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.535616 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 2
14:12:57.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.541322 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.626837 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 3
14:12:57.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.631870 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:57.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.772256 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 4
14:12:57.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.777426 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:57.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.986052 21669 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 6
14:12:57.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.992985 21669 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:58.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.060905 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 5
14:12:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:58.065182 21594 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.2.188:36993: Network error: Client connection negotiation failed: client connection to 127.21.2.188:36993: connect: Connection refused (error 111)
14:12:58.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.108567 21681 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:58.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:57.109453 21684 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:12:58.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:57.110280 21671 server_base.cc:1034] running on GCE node
14:12:58.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.198226 21671 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:12:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.200966 21671 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:12:58.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.202353 21671 hybrid_clock.cc:648] HybridClock initialized: now 1714659178202326 us; error 45 us; skew 500 ppm
14:12:58.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.203241 21671 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:12:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.209518 21671 webserver.cc:460] Webserver started at http://127.21.2.188:41879/ using document root <none> and password file <none>
14:12:58.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.210522 21671 fs_manager.cc:359] Metadata directory not provided
14:12:58.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.210750 21671 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:12:58.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.211254 21671 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:12:58.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.216292 21671 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/data/instance:
14:12:58.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7dbc595a3a284e2cb95c56797ff04fba"
14:12:58.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:58 on dist-test-slave-smf8"
14:12:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.217711 21671 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/wal/instance:
14:12:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7dbc595a3a284e2cb95c56797ff04fba"
14:12:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:58 on dist-test-slave-smf8"
14:12:58.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.225503 21671 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
14:12:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.231105 21696 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:12:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.232151 21671 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:12:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.232498 21671 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/wal
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7dbc595a3a284e2cb95c56797ff04fba"
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:12:58 on dist-test-slave-smf8"
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.232908 21671 fs_report.cc:383] FS layout report
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/wal
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/wal
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/data/data
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:12:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:12:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:12:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:12:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:12:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:12:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:12:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:12:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:12:58.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.309075 21671 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:12:58.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.310626 21671 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:12:58.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.311147 21671 kserver.cc:163] Server-wide thread pool size limit: 3276
14:12:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.379559 21671 rpc_server.cc:307] RPC server started. Bound to: 127.21.2.188:36993
14:12:58.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.379645 21747 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.2.188:36993 every 8 connection(s)
14:12:58.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.382429 21671 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/data/info.pb
14:12:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.387573 21748 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:12:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.388124 21514 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 21671
14:12:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.388454 21514 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/master-2/wal/instance
14:12:58.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.406548 21748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } has no permanent_uuid. Determining permanent_uuid...
14:12:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.432834 21748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } has no permanent_uuid. Determining permanent_uuid...
14:12:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.441975 21748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } has no permanent_uuid. Determining permanent_uuid...
14:12:58.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.457010 21748 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba: Bootstrap starting.
14:12:58.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.462807 21748 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba: Neither blocks nor log segments found. Creating new log.
14:12:58.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.464673 21748 log.cc:824] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba: Log is configured to *not* fsync() on all Append() calls
14:12:58.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.469257 21748 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba: No bootstrap required, opened a new log
14:12:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.489323 21748 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.491129 21748 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:12:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.491364 21748 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7dbc595a3a284e2cb95c56797ff04fba, State: Initialized, Role: FOLLOWER
14:12:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.492095 21748 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.494421 21757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.495079 21757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: This master's current role is: FOLLOWER
14:12:58.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.495782 21748 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: configured and running, proceeding with master startup.
14:12:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:58.520859 21768 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:12:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:58.521245 21768 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:12:58.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.515475 21671 master_runner.cc:186] Error getting master registration for 127.21.2.190:46093: OK, instance_id { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" instance_seqno: 1714659175002081 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:12:58.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.540000 21671 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:12:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:58.566222 21702 tablet.cc:2369] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:12:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.617570 21594 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 6
14:12:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.634891 21594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: Bootstrap starting.
14:12:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.639899 21594 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: Neither blocks nor log segments found. Creating new log.
14:12:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.641785 21594 log.cc:824] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: Log is configured to *not* fsync() on all Append() calls
14:12:58.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.646458 21594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: No bootstrap required, opened a new log
14:12:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.667587 21594 raft_consensus.cc:348] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.668229 21594 raft_consensus.cc:374] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:12:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.668424 21594 raft_consensus.cc:729] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 02b16b44ded54d9c892cdd338ea6149e, State: Initialized, Role: FOLLOWER
14:12:58.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.669395 21594 consensus_queue.cc:260] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.672328 21772 sys_catalog.cc:455] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.674015 21772 sys_catalog.cc:458] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [sys.catalog]: This master's current role is: FOLLOWER
14:12:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.674345 21594 sys_catalog.cc:564] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [sys.catalog]: configured and running, proceeding with master startup.
14:12:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:58.708552 21548 tablet.cc:2369] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:12:58.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:58.736573 21783 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:12:58.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:58.737368 21783 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:12:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.709798 21526 master_runner.cc:186] Error getting master registration for 127.21.2.189:33785: OK, instance_id { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" instance_seqno: 1714659176674729 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:12:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.758646 21526 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:12:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.879217 21757 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:12:58.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.879772 21757 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.882404 21757 leader_election.cc:288] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 02b16b44ded54d9c892cdd338ea6149e (127.21.2.190:46093), b5fda3b8cbfb415c8444b4db8bad6284 (127.21.2.189:33785)
14:12:58.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.884889 21569 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7dbc595a3a284e2cb95c56797ff04fba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02b16b44ded54d9c892cdd338ea6149e" is_pre_election: true
14:12:58.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.885861 21569 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7dbc595a3a284e2cb95c56797ff04fba in term 0.
14:12:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.887364 21700 leader_election.cc:302] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02b16b44ded54d9c892cdd338ea6149e, 7dbc595a3a284e2cb95c56797ff04fba; no voters:
14:12:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.888772 21757 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 0 FOLLOWER]: Leader pre-election won for term 1
14:12:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.889123 21757 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:12:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.889564 21757 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 0 FOLLOWER]: Advancing to term 1
14:12:58.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.896896 21757 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.899196 21757 leader_election.cc:288] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [CANDIDATE]: Term 1 election: Requested vote from peers 02b16b44ded54d9c892cdd338ea6149e (127.21.2.190:46093), b5fda3b8cbfb415c8444b4db8bad6284 (127.21.2.189:33785)
14:12:58.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.899753 21569 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7dbc595a3a284e2cb95c56797ff04fba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "02b16b44ded54d9c892cdd338ea6149e"
14:12:58.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.900343 21569 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [term 0 FOLLOWER]: Advancing to term 1
14:12:58.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.910663 21569 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7dbc595a3a284e2cb95c56797ff04fba in term 1.
14:12:58.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.912065 21700 leader_election.cc:302] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 02b16b44ded54d9c892cdd338ea6149e, 7dbc595a3a284e2cb95c56797ff04fba; no voters:
14:12:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.919142 21757 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 1 FOLLOWER]: Leader election won for term 1
14:12:58.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.929692 21757 raft_consensus.cc:686] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [term 1 LEADER]: Becoming Leader. State: Replica: 7dbc595a3a284e2cb95c56797ff04fba, State: Running, Role: LEADER
14:12:58.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.930922 21757 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.942932 21785 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7dbc595a3a284e2cb95c56797ff04fba. Latest consensus state: current_term: 1 leader_uuid: "7dbc595a3a284e2cb95c56797ff04fba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.944134 21785 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: This master's current role is: LEADER
14:12:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.947391 21787 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
14:12:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.954752 21787 catalog_manager.cc:1484] Initializing Kudu cluster ID...
14:12:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.978940 21569 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [term 1 FOLLOWER]: Refusing update from remote peer 7dbc595a3a284e2cb95c56797ff04fba: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:12:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:58.982818 21785 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [LEADER]: Connected to new peer: Peer: permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:12:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.010826 21772 sys_catalog.cc:455] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7dbc595a3a284e2cb95c56797ff04fba. Latest consensus state: current_term: 1 leader_uuid: "7dbc595a3a284e2cb95c56797ff04fba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:59.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.011550 21772 sys_catalog.cc:458] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [sys.catalog]: This master's current role is: FOLLOWER
14:12:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.013087 21788 mvcc.cc:205] Tried to move back new op lower bound from 7023243997075337216 to 7023243996912410624. Current Snapshot: MvccSnapshot[applied={T|T < 7023243997075337216}]
14:12:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.027441 21757 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7dbc595a3a284e2cb95c56797ff04fba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.028223 21757 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: This master's current role is: LEADER
14:12:59.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.029783 21789 mvcc.cc:205] Tried to move back new op lower bound from 7023243997075337216 to 7023243996912410624. Current Snapshot: MvccSnapshot[applied={T|T < 7023243997075337216}]
14:12:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.036839 21772 sys_catalog.cc:455] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7dbc595a3a284e2cb95c56797ff04fba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.037621 21772 sys_catalog.cc:458] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e [sys.catalog]: This master's current role is: FOLLOWER
14:12:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.045087 21787 catalog_manager.cc:1347] Generated new cluster ID: 7881c5eff7dc46c3974fb75f9152efe3
14:12:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.045418 21787 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
14:12:59.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.053416 21669 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } attempt: 7
14:12:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.104115 21669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284: Bootstrap starting.
14:12:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.108830 21787 catalog_manager.cc:1370] Generated new certificate authority record
14:12:59.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.111482 21787 catalog_manager.cc:1504] Loading token signing keys...
14:12:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.120877 21669 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284: Neither blocks nor log segments found. Creating new log.
14:12:59.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.132365 21669 log.cc:824] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284: Log is configured to *not* fsync() on all Append() calls
14:12:59.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.150559 21669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284: No bootstrap required, opened a new log
14:12:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.171998 21787 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba: Generated new TSK 0
14:12:59.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.175262 21787 catalog_manager.cc:1514] Initializing in-progress tserver states...
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.218103 21669 raft_consensus.cc:348] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.218967 21669 raft_consensus.cc:374] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.219230 21669 raft_consensus.cc:729] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5fda3b8cbfb415c8444b4db8bad6284, State: Initialized, Role: FOLLOWER
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.219359 21514 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/wal
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/data
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/logs
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/data/info.pb
14:12:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.129:0
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.2.129
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.129
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:12:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.220181 21669 consensus_queue.cc:260] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } }
14:12:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.243682 21795 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.245862 21795 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [sys.catalog]: This master's current role is: FOLLOWER
14:12:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.253749 21669 sys_catalog.cc:564] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [sys.catalog]: configured and running, proceeding with master startup.
14:12:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.253955 21644 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7dbc595a3a284e2cb95c56797ff04fba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" is_pre_election: true
14:12:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.254946 21644 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7dbc595a3a284e2cb95c56797ff04fba in term 0.
14:12:59.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.256242 21643 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7dbc595a3a284e2cb95c56797ff04fba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b5fda3b8cbfb415c8444b4db8bad6284"
14:12:59.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.256927 21643 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [term 0 FOLLOWER]: Advancing to term 1
14:12:59.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.264745 21643 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7dbc595a3a284e2cb95c56797ff04fba in term 1.
14:12:59.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.266347 21642 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [term 1 FOLLOWER]: Refusing update from remote peer 7dbc595a3a284e2cb95c56797ff04fba: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:12:59.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.274199 21785 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [LEADER]: Connected to new peer: Peer: permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:12:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.332228 21795 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7dbc595a3a284e2cb95c56797ff04fba. Latest consensus state: current_term: 1 leader_uuid: "7dbc595a3a284e2cb95c56797ff04fba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.333034 21795 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [sys.catalog]: This master's current role is: FOLLOWER
14:12:59.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.334594 21805 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7dbc595a3a284e2cb95c56797ff04fba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.336314 21805 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284 [sys.catalog]: This master's current role is: FOLLOWER
14:12:59.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.364435 21812 catalog_manager.cc:1259] Loaded cluster ID: 7881c5eff7dc46c3974fb75f9152efe3
14:12:59.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.366055 21812 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284: loading cluster ID for follower catalog manager: success
14:12:59.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.379534 21812 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284: acquiring CA information for follower catalog manager: success
14:12:59.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.389400 21812 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b5fda3b8cbfb415c8444b4db8bad6284: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:12:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.743237 21783 catalog_manager.cc:1259] Loaded cluster ID: 7881c5eff7dc46c3974fb75f9152efe3
14:12:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.743534 21783 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: loading cluster ID for follower catalog manager: success
14:12:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.758740 21783 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: acquiring CA information for follower catalog manager: success
14:12:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.764094 21783 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 02b16b44ded54d9c892cdd338ea6149e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:12:59.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:59.797008 21796 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:12:59.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:59.797670 21796 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:12:59.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:59.798434 21796 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:12:59.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:59.853631 21796 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:12:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:59.855031 21796 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.2.129
14:12:59.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.862886 21785 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7dbc595a3a284e2cb95c56797ff04fba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "02b16b44ded54d9c892cdd338ea6149e" member_type: VOTER last_known_addr { host: "127.21.2.190" port: 46093 } } peers { permanent_uuid: "b5fda3b8cbfb415c8444b4db8bad6284" member_type: VOTER last_known_addr { host: "127.21.2.189" port: 33785 } } peers { permanent_uuid: "7dbc595a3a284e2cb95c56797ff04fba" member_type: VOTER last_known_addr { host: "127.21.2.188" port: 36993 } } }
14:12:59.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.863967 21785 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7dbc595a3a284e2cb95c56797ff04fba [sys.catalog]: This master's current role is: LEADER
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.915511 21796 tablet_server_runner.cc:78] Tablet server non-default flags:
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/data
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/wal
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.129:0
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/data/info.pb
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.129
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:12:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.2.129
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/logs
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:12:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.917119 21796 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:12:59.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:12:59.919245 21796 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:12:59.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:59.960558 21822 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:01.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:59.973347 21825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:12:59.963426 21821 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:01.639423 21823 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1678 milliseconds
14:13:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.639532 21796 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:13:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.641229 21796 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:01.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.654893 21796 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.656682 21796 hybrid_clock.cc:648] HybridClock initialized: now 1714659181656601 us; error 67 us; skew 500 ppm
14:13:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.657809 21796 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:01.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.689652 21796 webserver.cc:460] Webserver started at http://127.21.2.129:44595/ using document root <none> and password file <none>
14:13:01.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.704591 21796 fs_manager.cc:359] Metadata directory not provided
14:13:01.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.705003 21796 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:01.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.705626 21796 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:13:01.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.715664 21796 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/data/instance:
14:13:01.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f2e223d2770844faade11cf60808d027"
14:13:01.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:01 on dist-test-slave-smf8"
14:13:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.722224 21796 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/wal/instance:
14:13:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f2e223d2770844faade11cf60808d027"
14:13:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:01 on dist-test-slave-smf8"
14:13:01.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.751197 21796 fs_manager.cc:691] Time spent creating directory manager: real 0.027s user 0.007s sys 0.004s
14:13:01.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.767913 21835 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:01.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.769371 21796 fs_manager.cc:725] Time spent opening block manager: real 0.013s user 0.001s sys 0.008s
14:13:01.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.769788 21796 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/wal
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f2e223d2770844faade11cf60808d027"
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:01 on dist-test-slave-smf8"
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.770254 21796 fs_report.cc:383] FS layout report
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/wal
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/wal
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/data/data
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.871616 21796 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.873694 21796 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.874316 21796 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:01.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.883910 21796 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
14:13:01.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.897234 21796 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:13:01.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.897540 21796 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:13:01.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.897832 21796 ts_tablet_manager.cc:590] Registered 0 tablets
14:13:01.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:01.898082 21796 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:13:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.250111 21796 rpc_server.cc:307] RPC server started. Bound to: 127.21.2.129:32797
14:13:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.250265 21947 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.2.129:32797 every 8 connection(s)
14:13:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.253593 21796 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/data/info.pb
14:13:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.255211 21514 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 21796
14:13:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.255676 21514 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-0/wal/instance
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.294425 21514 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/wal
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/data
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/logs
14:13:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/data/info.pb
14:13:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.130:0
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.2.130
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.130
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:13:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.342209 21949 heartbeater.cc:347] Connected to a master server at 127.21.2.189:33785
14:13:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.342696 21949 heartbeater.cc:464] Registering TS with master...
14:13:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.343948 21949 heartbeater.cc:511] Master 127.21.2.189:33785 requested a full tablet report, sending...
14:13:02.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.348752 21634 ts_manager.cc:194] Registered new tserver with Master: f2e223d2770844faade11cf60808d027 (127.21.2.129:32797)
14:13:02.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.364496 21952 heartbeater.cc:347] Connected to a master server at 127.21.2.190:46093
14:13:02.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.364883 21952 heartbeater.cc:464] Registering TS with master...
14:13:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.365829 21952 heartbeater.cc:511] Master 127.21.2.190:46093 requested a full tablet report, sending...
14:13:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.367172 21948 heartbeater.cc:347] Connected to a master server at 127.21.2.188:36993
14:13:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.367512 21948 heartbeater.cc:464] Registering TS with master...
14:13:02.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.368806 21948 heartbeater.cc:511] Master 127.21.2.188:36993 requested a full tablet report, sending...
14:13:02.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.369056 21558 ts_manager.cc:194] Registered new tserver with Master: f2e223d2770844faade11cf60808d027 (127.21.2.129:32797)
14:13:02.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.372398 21713 ts_manager.cc:194] Registered new tserver with Master: f2e223d2770844faade11cf60808d027 (127.21.2.129:32797)
14:13:02.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.375978 21713 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.2.129:37315
14:13:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:02.697436 21955 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:02.697899 21955 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:02.698418 21955 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:02.730767 21955 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:02.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:02.731559 21955 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.2.130
14:13:02.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.764463 21955 tablet_server_runner.cc:78] Tablet server non-default flags:
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/data
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/wal
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.130:0
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/data/info.pb
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.130
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.2.130
14:13:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/logs
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.765607 21955 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:02.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:02.767387 21955 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:02.785321 21965 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:03.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.380100 21948 heartbeater.cc:503] Master 127.21.2.188:36993 was elected leader, sending a full tablet report...
14:13:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:02.785614 21964 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:02.788018 21967 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:03.916110 21966 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1129 milliseconds
14:13:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.916216 21955 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:13:03.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.917547 21955 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.920470 21955 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.921947 21955 hybrid_clock.cc:648] HybridClock initialized: now 1714659183921904 us; error 63 us; skew 500 ppm
14:13:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.922724 21955 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:03.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.929100 21955 webserver.cc:460] Webserver started at http://127.21.2.130:46711/ using document root <none> and password file <none>
14:13:03.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.930002 21955 fs_manager.cc:359] Metadata directory not provided
14:13:03.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.930198 21955 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:03.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.930591 21955 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:13:03.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.935397 21955 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/data/instance:
14:13:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce9389f75cd04c1591b93b0922e644eb"
14:13:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:03 on dist-test-slave-smf8"
14:13:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.936498 21955 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/wal/instance:
14:13:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce9389f75cd04c1591b93b0922e644eb"
14:13:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:03 on dist-test-slave-smf8"
14:13:03.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.943619 21955 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
14:13:03.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.949877 21976 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.950999 21955 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:13:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.951304 21955 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/wal
14:13:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce9389f75cd04c1591b93b0922e644eb"
14:13:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:03 on dist-test-slave-smf8"
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:03.951625 21955 fs_report.cc:383] FS layout report
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/wal
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/wal
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/data/data
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:04.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.008024 21955 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:04.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.009605 21955 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:04.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.010051 21955 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.012619 21955 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
14:13:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.016990 21955 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:13:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.017210 21955 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:13:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.017411 21955 ts_tablet_manager.cc:590] Registered 0 tablets
14:13:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.017560 21955 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:13:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.180181 21955 rpc_server.cc:307] RPC server started. Bound to: 127.21.2.130:42143
14:13:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.180289 22088 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.2.130:42143 every 8 connection(s)
14:13:04.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.182809 21955 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/data/info.pb
14:13:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.190135 21514 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 21955
14:13:04.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.190892 21514 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-1/wal/instance
14:13:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.215588 21514 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/wal
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/data
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/logs
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/data/info.pb
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:13:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.131:0
14:13:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.2.131
14:13:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.131
14:13:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:13:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:13:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:13:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.241919 22092 heartbeater.cc:347] Connected to a master server at 127.21.2.190:46093
14:13:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.242470 22092 heartbeater.cc:464] Registering TS with master...
14:13:04.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.243643 22092 heartbeater.cc:511] Master 127.21.2.190:46093 requested a full tablet report, sending...
14:13:04.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.247632 21558 ts_manager.cc:194] Registered new tserver with Master: ce9389f75cd04c1591b93b0922e644eb (127.21.2.130:42143)
14:13:04.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.248677 22089 heartbeater.cc:347] Connected to a master server at 127.21.2.188:36993
14:13:04.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.249002 22089 heartbeater.cc:464] Registering TS with master...
14:13:04.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.249807 22089 heartbeater.cc:511] Master 127.21.2.188:36993 requested a full tablet report, sending...
14:13:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.251984 21713 ts_manager.cc:194] Registered new tserver with Master: ce9389f75cd04c1591b93b0922e644eb (127.21.2.130:42143)
14:13:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.253700 21713 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.2.130:57819
14:13:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.254643 22090 heartbeater.cc:347] Connected to a master server at 127.21.2.189:33785
14:13:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.254992 22090 heartbeater.cc:464] Registering TS with master...
14:13:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.255867 22090 heartbeater.cc:511] Master 127.21.2.189:33785 requested a full tablet report, sending...
14:13:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.258467 21634 ts_manager.cc:194] Registered new tserver with Master: ce9389f75cd04c1591b93b0922e644eb (127.21.2.130:42143)
14:13:04.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:04.587816 22099 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:04.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:04.588378 22099 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:04.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:04.588876 22099 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:04.619629 22099 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:04.620455 22099 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.2.131
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.656109 22099 tablet_server_runner.cc:78] Tablet server non-default flags:
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.2.148:38339
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/data
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/wal
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.2.131:0
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/data/info.pb
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.2.131
14:13:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.2.190:46093,127.21.2.189:33785,127.21.2.188:36993
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.2.131
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/logs
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:04.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.657281 22099 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:04.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.658782 22099 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:04.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:04.681147 22108 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:05.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.261509 22089 heartbeater.cc:503] Master 127.21.2.188:36993 was elected leader, sending a full tablet report...
14:13:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:04.683036 22105 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:04.681788 22106 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:04.683274 22099 server_base.cc:1034] running on GCE node
14:13:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.787014 22099 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.789330 22099 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:05.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.790699 22099 hybrid_clock.cc:648] HybridClock initialized: now 1714659185790655 us; error 51 us; skew 500 ppm
14:13:05.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.791538 22099 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.797900 22099 webserver.cc:460] Webserver started at http://127.21.2.131:43473/ using document root <none> and password file <none>
14:13:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.798805 22099 fs_manager.cc:359] Metadata directory not provided
14:13:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.799021 22099 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.799531 22099 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:13:05.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.804373 22099 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/data/instance:
14:13:05.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "321cd892c95f422ebe59ca61676f9e2f"
14:13:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:05 on dist-test-slave-smf8"
14:13:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.805599 22099 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/wal/instance:
14:13:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "321cd892c95f422ebe59ca61676f9e2f"
14:13:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:05 on dist-test-slave-smf8"
14:13:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.812495 22099 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
14:13:05.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.818055 22117 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:05.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.819061 22099 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
14:13:05.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.819378 22099 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/wal
14:13:05.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "321cd892c95f422ebe59ca61676f9e2f"
14:13:05.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:05 on dist-test-slave-smf8"
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.819716 22099 fs_report.cc:383] FS layout report
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/wal
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/wal
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/data/data
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.874493 22099 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:05.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.876170 22099 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:05.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.876696 22099 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:05.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.879340 22099 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
14:13:05.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.883345 22099 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:13:05.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.883538 22099 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:13:05.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.883849 22099 ts_tablet_manager.cc:590] Registered 0 tablets
14:13:05.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:05.884074 22099 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:13:06.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.049434 22099 rpc_server.cc:307] RPC server started. Bound to: 127.21.2.131:44953
14:13:06.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.049510 22229 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.2.131:44953 every 8 connection(s)
14:13:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.052867 22099 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/data/info.pb
14:13:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.061883 21514 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 22099
14:13:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.062482 21514 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster1270344593256716084/ts-2/wal/instance
14:13:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.102382 22230 heartbeater.cc:347] Connected to a master server at 127.21.2.188:36993
14:13:06.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.102921 22230 heartbeater.cc:464] Registering TS with master...
14:13:06.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.103500 22231 heartbeater.cc:347] Connected to a master server at 127.21.2.189:33785
14:13:06.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.103864 22231 heartbeater.cc:464] Registering TS with master...
14:13:06.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.104321 22230 heartbeater.cc:511] Master 127.21.2.188:36993 requested a full tablet report, sending...
14:13:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.105084 22231 heartbeater.cc:511] Master 127.21.2.189:33785 requested a full tablet report, sending...
14:13:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.106868 21713 ts_manager.cc:194] Registered new tserver with Master: 321cd892c95f422ebe59ca61676f9e2f (127.21.2.131:44953)
14:13:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.107846 21634 ts_manager.cc:194] Registered new tserver with Master: 321cd892c95f422ebe59ca61676f9e2f (127.21.2.131:44953)
14:13:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.108556 21713 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.2.131:42683
14:13:06.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.116794 22232 heartbeater.cc:347] Connected to a master server at 127.21.2.190:46093
14:13:06.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.117126 22232 heartbeater.cc:464] Registering TS with master...
14:13:06.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.117703 22232 heartbeater.cc:511] Master 127.21.2.190:46093 requested a full tablet report, sending...
14:13:06.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.120229 21559 ts_manager.cc:194] Registered new tserver with Master: 321cd892c95f422ebe59ca61676f9e2f (127.21.2.131:44953)
14:13:06.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.133827 21514 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
14:13:06.144 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:13:06.145 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:13:06.156 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.2.190"
port: 46093
}
bound_http_address {
host: "127.21.2.190"
port: 45393
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.2.189"
port: 33785
}
bound_http_address {
host: "127.21.2.189"
port: 35205
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.2.188"
port: 36993
}
bound_http_address {
host: "127.21.2.188"
port: 41879
}
}
}
14:13:06.160 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:13:06.162 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.21.2.129"
port: 32797
}
bound_http_address {
host: "127.21.2.129"
port: 44595
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.21.2.130"
port: 42143
}
bound_http_address {
host: "127.21.2.130"
port: 46711
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.21.2.131"
port: 44953
}
bound_http_address {
host: "127.21.2.131"
port: 43473
}
}
}
14:13:06.163 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:13:06.339 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:13:06.462 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQ4kZtv/test-tmp
14:13:06.463 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:13:06.463 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQ4kZtv/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:13:06.467 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:13:06.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:06.802 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:13:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.805071 22241 mini_oidc.cc:152] Starting JWKS server
14:13:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.808501 22241 webserver.cc:460] Webserver started at http://127.0.0.1:39721/ using document root <none> and password file <none>
14:13:06.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.809005 22241 mini_oidc.cc:189] Starting OIDC Discovery server
14:13:06.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:06.810825 22241 webserver.cc:460] Webserver started at http://127.0.0.1:36753/ using document root <none> and password file <none>
14:13:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:13:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/principal' for realm 'KRBTEST.COM',
14:13:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:13:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:06 dist-test-slave-smf8 krb5kdc[22253](info): setting up network...
14:13:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:13:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:06 dist-test-slave-smf8 krb5kdc[22253](info): set up 2 sockets
14:13:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:06 dist-test-slave-smf8 krb5kdc[22253](info): commencing operation
14:13:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:13:07.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:07.113350 22230 heartbeater.cc:503] Master 127.21.2.188:36993 was elected leader, sending a full tablet report...
14:13:07.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:07.852363 22241 mini_kdc.cc:121] Time spent starting KDC: real 1.003s user 0.005s sys 0.014s
14:13:07.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:13:07.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:13:07.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:13:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:13:07.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:13:07.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:13:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:13:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:13:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:13:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:13:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/test-user.keytab.
14:13:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/test-user.keytab.
14:13:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:07 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659187, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:13:07.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:13:07Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
14:13:07.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:13:07Z Disabled control of system clock
14:13:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.21.184.126@KRBTEST.COM; defaulting to no policy
14:13:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:13:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.21.184.126@KRBTEST.COM" created.
14:13:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:13:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.184.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.126.keytab.
14:13:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.184.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.126.keytab.
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:08.039295 22241 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/logs
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/info.pb
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.126:39787
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.126
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.126.keytab
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.126
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:13:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.342618 22267 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.343278 22267 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:08.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.343955 22267 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:08.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.375288 22267 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:13:08.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.375610 22267 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.376044 22267 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.376363 22267 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.376749 22267 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.377055 22267 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.377283 22267 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39721/jwks.json
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:08.420015 22267 master_runner.cc:386] Master server non-default flags:
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.126.keytab
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.126
14:13:08.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.126:39787
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/info.pb
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.126
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/logs
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:08.421533 22267 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:08.422883 22267 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.440711 22276 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.441391 22273 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:08.442193 22274 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:09.667956 22275 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1226 milliseconds
14:13:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.668076 22267 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:13:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.669447 22267 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.672399 22267 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.673826 22267 hybrid_clock.cc:648] HybridClock initialized: now 1714659189673779 us; error 79 us; skew 500 ppm
14:13:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.674635 22267 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:09 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659189, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.680569 22267 init.cc:377] Logged in from keytab as kudu/127.21.184.126@KRBTEST.COM (short username kudu)
14:13:09.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.685961 22267 webserver.cc:460] Webserver started at http://127.21.184.126:41921/ using document root <none> and password file <none>
14:13:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.686895 22267 fs_manager.cc:359] Metadata directory not provided
14:13:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.687095 22267 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.687494 22267 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:13:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.692476 22267 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/instance:
14:13:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79797cf15ca24cc58d04b4e03ffcdcff"
14:13:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:09 on dist-test-slave-smf8"
14:13:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.693660 22267 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal/instance:
14:13:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79797cf15ca24cc58d04b4e03ffcdcff"
14:13:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:09 on dist-test-slave-smf8"
14:13:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.702601 22267 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.001s sys 0.008s
14:13:09.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.708889 22284 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.709962 22267 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
14:13:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.710273 22267 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79797cf15ca24cc58d04b4e03ffcdcff"
14:13:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:09 on dist-test-slave-smf8"
14:13:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.710574 22267 fs_report.cc:383] FS layout report
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/data
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.767721 22267 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.773315 22267 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:09.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.773888 22267 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:09.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.852758 22267 rpc_server.cc:307] RPC server started. Bound to: 127.21.184.126:39787
14:13:09.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.852836 22336 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.184.126:39787 every 8 connection(s)
14:13:09.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.855643 22267 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/info.pb
14:13:09.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.860965 22337 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.864825 22241 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 22267
14:13:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.865392 22241 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal/instance
14:13:09.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.878450 22337 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } has no permanent_uuid. Determining permanent_uuid...
14:13:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.21.184.125@KRBTEST.COM; defaulting to no policy
14:13:09.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:13:09.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.21.184.125@KRBTEST.COM" created.
14:13:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:09 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659189, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.126@KRBTEST.COM for kudu/127.21.184.126@KRBTEST.COM
14:13:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.914675 22337 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } has no permanent_uuid. Determining permanent_uuid...
14:13:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:13:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.184.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.125.keytab.
14:13:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.184.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.125.keytab.
14:13:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:09.917413 22286 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.917860 22241 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/logs
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/info.pb
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.125:44107
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.125
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.125.keytab
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.125
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:09.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:13:09.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:09.923636 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.125:44107: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:09.973201 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } attempt: 1
14:13:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:09.976804 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.125:44107: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:10.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:10.045253 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } attempt: 2
14:13:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.050705 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.125:44107: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:10.142217 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } attempt: 3
14:13:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.146651 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.125:44107: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.243322 22344 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.243861 22344 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.244453 22344 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.282147 22344 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:13:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.282373 22344 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.282631 22344 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:10.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.282826 22344 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.283067 22344 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.283427 22344 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.283669 22344 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39721/jwks.json
14:13:10.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:10.290086 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } attempt: 4
14:13:10.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.294088 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.125:44107: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:10.321890 22344 master_runner.cc:386] Master server non-default flags:
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:13:10.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.125.keytab
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.125
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.125:44107
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/info.pb
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.125
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/logs
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:10.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:13:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:13:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:13:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:10.323699 22344 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:10.325227 22344 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.342710 22350 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:10.593520 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } attempt: 5
14:13:10.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.597709 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.125:44107: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.145115 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } attempt: 6
14:13:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:11.152541 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.125:44107: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.345494 22351 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:10.348448 22344 server_base.cc:1034] running on GCE node
14:13:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:10.349113 22353 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:11.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.496168 22344 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:11.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.498817 22344 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.500170 22344 hybrid_clock.cc:648] HybridClock initialized: now 1714659191500150 us; error 48 us; skew 500 ppm
14:13:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.501075 22344 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:11 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659191, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.507725 22344 init.cc:377] Logged in from keytab as kudu/127.21.184.125@KRBTEST.COM (short username kudu)
14:13:11.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.514307 22344 webserver.cc:460] Webserver started at http://127.21.184.125:39859/ using document root <none> and password file <none>
14:13:11.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.515252 22344 fs_manager.cc:359] Metadata directory not provided
14:13:11.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.515507 22344 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.515903 22344 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:13:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.520864 22344 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/instance:
14:13:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0824e38f238d4f93b0fb9d1eb74d0520"
14:13:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:11 on dist-test-slave-smf8"
14:13:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.521978 22344 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal/instance:
14:13:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0824e38f238d4f93b0fb9d1eb74d0520"
14:13:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:11 on dist-test-slave-smf8"
14:13:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.529611 22344 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.007s
14:13:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.534940 22363 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.536072 22344 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:13:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.536378 22344 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0824e38f238d4f93b0fb9d1eb74d0520"
14:13:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:11 on dist-test-slave-smf8"
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.536737 22344 fs_report.cc:383] FS layout report
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/data
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.590162 22344 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.596038 22344 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.596520 22344 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.692252 22344 rpc_server.cc:307] RPC server started. Bound to: 127.21.184.125:44107
14:13:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.692328 22414 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.184.125:44107 every 8 connection(s)
14:13:11.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.695947 22344 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/info.pb
14:13:11.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.703881 22415 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.704551 22241 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 22344
14:13:11.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.705080 22241 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal/instance
14:13:11.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.21.184.124@KRBTEST.COM; defaulting to no policy
14:13:11.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:13:11.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.21.184.124@KRBTEST.COM" created.
14:13:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.724062 22415 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } has no permanent_uuid. Determining permanent_uuid...
14:13:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:11 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659191, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.125@KRBTEST.COM for kudu/127.21.184.126@KRBTEST.COM
14:13:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:13:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.184.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.124.keytab.
14:13:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.21.184.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.124.keytab.
14:13:11.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.754499 22241 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/logs
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/info.pb
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.124:36891
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.124
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.124.keytab
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.124
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:13:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.765781 22415 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } has no permanent_uuid. Determining permanent_uuid...
14:13:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:11 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659191, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.125@KRBTEST.COM for kudu/127.21.184.125@KRBTEST.COM
14:13:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.790405 22415 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } has no permanent_uuid. Determining permanent_uuid...
14:13:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:11.792920 22367 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:11.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:11.797489 22415 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.846951 22415 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 1
14:13:11.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:11.851248 22415 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:11.919708 22415 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 2
14:13:11.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:11.923707 22415 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.015102 22415 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 3
14:13:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.018951 22415 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.105654 22421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:12.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.106312 22421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:12.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.106868 22421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.139189 22421 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:13:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.139513 22421 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.139905 22421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.140126 22421 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.140321 22421 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.140504 22421 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.140687 22421 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39721/jwks.json
14:13:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.162422 22415 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 4
14:13:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.166646 22415 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.178930 22421 master_runner.cc:386] Master server non-default flags:
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.124.keytab
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.124
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.124:36891
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/info.pb
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.124
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/logs
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.180033 22421 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.181283 22421 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:12.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.202242 22430 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.212976 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } attempt: 7
14:13:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:12 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659189, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.126@KRBTEST.COM for kudu/127.21.184.125@KRBTEST.COM
14:13:12.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.244160 22337 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } has no permanent_uuid. Determining permanent_uuid...
14:13:12.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.259603 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.318341 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 1
14:13:12.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.323468 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.405045 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 2
14:13:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.412775 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.466058 22415 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 5
14:13:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.471535 22415 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.498207 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 3
14:13:12.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.503376 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.643774 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 4
14:13:12.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.648957 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.932368 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 5
14:13:12.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.937276 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:13.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.018963 22415 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 6
14:13:13.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:13.029975 22415 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:13.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.202836 22432 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:13.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:12.203423 22429 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:12.204139 22421 server_base.cc:1034] running on GCE node
14:13:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.367820 22421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:13.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.370608 22421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.371980 22421 hybrid_clock.cc:648] HybridClock initialized: now 1714659193371943 us; error 42 us; skew 500 ppm
14:13:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.372820 22421 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659193, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.379415 22421 init.cc:377] Logged in from keytab as kudu/127.21.184.124@KRBTEST.COM (short username kudu)
14:13:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.386029 22421 webserver.cc:460] Webserver started at http://127.21.184.124:45265/ using document root <none> and password file <none>
14:13:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.387284 22421 fs_manager.cc:359] Metadata directory not provided
14:13:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.387548 22421 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.388078 22421 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
14:13:13.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.396114 22421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/instance:
14:13:13.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc78ebecb52427d96121b13b64f3c52"
14:13:13.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:13 on dist-test-slave-smf8"
14:13:13.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.397694 22421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal/instance:
14:13:13.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc78ebecb52427d96121b13b64f3c52"
14:13:13.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:13 on dist-test-slave-smf8"
14:13:13.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.407706 22421 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
14:13:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.414497 22446 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.415627 22421 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.415962 22421 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc78ebecb52427d96121b13b64f3c52"
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:13 on dist-test-slave-smf8"
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.416311 22421 fs_report.cc:383] FS layout report
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/data
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.468770 22421 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.473837 22421 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.474248 22421 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:13.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.489660 22337 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } attempt: 6
14:13:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:13.499979 22337 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.21.184.124:36891: Network error: Client connection negotiation failed: client connection to 127.21.184.124:36891: connect: Connection refused (error 111)
14:13:13.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.571877 22421 rpc_server.cc:307] RPC server started. Bound to: 127.21.184.124:36891
14:13:13.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.571990 22499 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.184.124:36891 every 8 connection(s)
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.574427 22421 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/info.pb
14:13:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.579913 22500 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.582262 22241 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 22421
14:13:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.582603 22241 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal/instance
14:13:13.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.596897 22500 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } has no permanent_uuid. Determining permanent_uuid...
14:13:13.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659193, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.124@KRBTEST.COM for kudu/127.21.184.126@KRBTEST.COM
14:13:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659187, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.21.184.124@KRBTEST.COM
14:13:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.633846 22500 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } has no permanent_uuid. Determining permanent_uuid...
14:13:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659193, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.124@KRBTEST.COM for kudu/127.21.184.125@KRBTEST.COM
14:13:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659187, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.21.184.125@KRBTEST.COM
14:13:13.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.662024 22500 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } has no permanent_uuid. Determining permanent_uuid...
14:13:13.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659193, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.124@KRBTEST.COM for kudu/127.21.184.124@KRBTEST.COM
14:13:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659187, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.21.184.126@KRBTEST.COM
14:13:13.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.692605 22500 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: Bootstrap starting.
14:13:13.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.700271 22500 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: Neither blocks nor log segments found. Creating new log.
14:13:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.703437 22241 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
14:13:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.704023 22500 log.cc:824] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: Log is configured to *not* fsync() on all Append() calls
14:13:13.714 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:13.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:13:13.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.715198 22500 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: No bootstrap required, opened a new log
14:13:13.735 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:13:13.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:13:13.740 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.184.126"
port: 39787
}
bound_http_address {
host: "127.21.184.126"
port: 41921
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.184.125"
port: 44107
}
bound_http_address {
host: "127.21.184.125"
port: 39859
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.184.124"
port: 36891
}
bound_http_address {
host: "127.21.184.124"
port: 45265
}
}
}
14:13:13.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:13:13.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:13.741 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:13:13.742 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:13:13.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.748495 22500 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.749248 22500 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.749430 22500 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bc78ebecb52427d96121b13b64f3c52, State: Initialized, Role: FOLLOWER
14:13:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.750196 22500 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.753154 22511 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } } }
14:13:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.754050 22511 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [sys.catalog]: This master's current role is: FOLLOWER
14:13:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.755074 22500 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [sys.catalog]: configured and running, proceeding with master startup.
14:13:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659193, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:13.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:13:13.780 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:13.786 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc
14:13:13.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:13.788823 22523 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:13.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:13.789371 22523 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:13.794 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:13:13.795 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc
14:13:13.797 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:13:13.803 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:13:13.806 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:13:13.806 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:13:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.784010 22421 master_runner.cc:186] Error getting master registration for 127.21.184.126:39787: OK, instance_id { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" instance_seqno: 1714659189827521 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:13:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.816385 22421 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:13.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:13 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659193, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:13.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:13:13.848 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:13.848 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc
14:13:13.850 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:13:13.850 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.184.124:36891
14:13:13.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:13:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.853333 22241 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 22421
14:13:13.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:13.875 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.184.125:44107
14:13:13.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:13:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.876086 22241 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 22344
14:13:13.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:13.897 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.21.184.126:39787
14:13:13.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:13:13.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.898209 22241 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 22267
14:13:13.921 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:13.921 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.184.124:36891
14:13:13.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:13:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:13.925180 22241 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/logs
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/info.pb
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.124:36891
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.124
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45265
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.124.keytab
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.124
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:13.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:13:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.229940 22527 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.230648 22527 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.231348 22527 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.268128 22527 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:13:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.268472 22527 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.268860 22527 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.269085 22527 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.269274 22527 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.269460 22527 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.269598 22527 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39721/jwks.json
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:14.307530 22527 master_runner.cc:386] Master server non-default flags:
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.124.keytab
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.124
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.124:36891
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/info.pb
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.124
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45265
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/logs
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:14.308669 22527 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:14.309666 22527 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:14.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.327281 22536 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.327809 22534 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:14.328264 22533 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:14.329308 22527 server_base.cc:1034] running on GCE node
14:13:15.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.465520 22527 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.468848 22527 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.470310 22527 hybrid_clock.cc:648] HybridClock initialized: now 1714659195470260 us; error 60 us; skew 500 ppm
14:13:15.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.471093 22527 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:15 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659195, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.476990 22527 init.cc:377] Logged in from keytab as kudu/127.21.184.124@KRBTEST.COM (short username kudu)
14:13:15.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.482766 22527 webserver.cc:460] Webserver started at http://127.21.184.124:45265/ using document root <none> and password file <none>
14:13:15.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.483686 22527 fs_manager.cc:359] Metadata directory not provided
14:13:15.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.483894 22527 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.492148 22527 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
14:13:15.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.497221 22546 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.498343 22527 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
14:13:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.498642 22527 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc78ebecb52427d96121b13b64f3c52"
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:13 on dist-test-slave-smf8"
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.500973 22527 fs_report.cc:383] FS layout report
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/wal
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/data
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.565296 22527 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:15.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.570732 22527 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:15.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.571175 22527 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.641291 22527 rpc_server.cc:307] RPC server started. Bound to: 127.21.184.124:36891
14:13:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.641402 22597 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.184.124:36891 every 8 connection(s)
14:13:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.643981 22527 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-2/data/info.pb
14:13:15.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.646888 22241 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 22527
14:13:15.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:15.648 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.184.125:44107
14:13:15.648 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.650063 22241 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/logs
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/info.pb
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.125:44107
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.125
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39859
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.125.keytab
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.125
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:13:15.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.659026 22598 sys_catalog.cc:263] Verifying existing consensus state
14:13:15.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.664487 22598 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: Bootstrap starting.
14:13:15.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.697710 22598 log.cc:824] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: Log is configured to *not* fsync() on all Append() calls
14:13:15.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.702596 22598 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:13:15.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.703570 22598 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: Bootstrap complete.
14:13:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.729182 22598 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.731418 22598 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.731719 22598 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bc78ebecb52427d96121b13b64f3c52, State: Initialized, Role: FOLLOWER
14:13:15.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.732609 22598 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.735597 22602 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } } }
14:13:15.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.736707 22602 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [sys.catalog]: This master's current role is: FOLLOWER
14:13:15.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.737394 22598 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [sys.catalog]: configured and running, proceeding with master startup.
14:13:15.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.749444 22602 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.750056 22602 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.755218 22602 leader_election.cc:288] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787), 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107)
14:13:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:15.764230 22547 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:15.773384 22549 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787): Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111)
14:13:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:15.775532 22547 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107): Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.776193 22547 leader_election.cc:302] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2bc78ebecb52427d96121b13b64f3c52; no voters: 0824e38f238d4f93b0fb9d1eb74d0520, 79797cf15ca24cc58d04b4e03ffcdcff
14:13:15.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.777169 22602 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:13:15.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:15.789788 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:15.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:15.790189 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.774690 22527 master_runner.cc:186] Error getting master registration for 127.21.184.126:39787: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111),
14:13:15.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:15.800755 22527 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:15.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:15.826138 22552 tablet.cc:2369] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:13:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.021945 22600 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.022477 22600 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:16.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.023025 22600 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:16.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.075589 22600 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:13:16.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.075922 22600 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:16.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.076390 22600 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:16.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.076745 22600 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.077064 22600 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.077384 22600 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.077620 22600 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39721/jwks.json
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:16.120823 22600 master_runner.cc:386] Master server non-default flags:
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.125.keytab
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.125
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:16.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.125:44107
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/info.pb
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.125
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39859
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/logs
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:13:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:16.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:16.121973 22600 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:16.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:16.123363 22600 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.141739 22623 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.793291 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.793596 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:16.143843 22600 server_base.cc:1034] running on GCE node
14:13:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.142895 22626 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:17.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:16.142091 22624 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.368214 22600 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:17.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.371402 22600 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.373041 22600 hybrid_clock.cc:648] HybridClock initialized: now 1714659197372996 us; error 62 us; skew 500 ppm
14:13:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.373883 22600 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:17 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659197, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:17.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.384300 22600 init.cc:377] Logged in from keytab as kudu/127.21.184.125@KRBTEST.COM (short username kudu)
14:13:17.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.391060 22600 webserver.cc:460] Webserver started at http://127.21.184.125:39859/ using document root <none> and password file <none>
14:13:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.392027 22600 fs_manager.cc:359] Metadata directory not provided
14:13:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.392259 22600 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:17.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.401118 22600 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
14:13:17.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.406352 22636 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.407506 22600 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
14:13:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.408082 22600 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0824e38f238d4f93b0fb9d1eb74d0520"
14:13:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:11 on dist-test-slave-smf8"
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.411751 22600 fs_report.cc:383] FS layout report
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/data
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:17.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:17.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.477335 22600 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.482900 22600 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.483434 22600 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.571468 22600 rpc_server.cc:307] RPC server started. Bound to: 127.21.184.125:44107
14:13:17.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.571584 22687 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.184.125:44107 every 8 connection(s)
14:13:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.575075 22600 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/data/info.pb
14:13:17.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.584673 22241 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 22600
14:13:17.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:17.586 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.21.184.126:39787
14:13:17.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:13:17.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.587239 22688 sys_catalog.cc:263] Verifying existing consensus state
14:13:17.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 14:13:17.588294 22688 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.588959 22600 master.cc:537] Master@127.21.184.125:44107 shutting down...
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.588996 22241 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/logs
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/info.pb
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.126:39787
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.126
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41921
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.126.keytab
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.126
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:13:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.676079 22691 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.676882 22691 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:17.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.684182 22600 master.cc:552] Master@127.21.184.125:44107 shutdown complete.
14:13:17.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.687407 22691 leader_election.cc:288] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787), 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107)
14:13:17.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:13:17.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.712924 22547 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107): Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:17.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.714135 22549 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787): Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111)
14:13:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.718283 22549 leader_election.cc:302] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2bc78ebecb52427d96121b13b64f3c52; no voters: 0824e38f238d4f93b0fb9d1eb74d0520, 79797cf15ca24cc58d04b4e03ffcdcff
14:13:17.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:17.718911 22691 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:13:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.797672 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:17.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.798022 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.934723 22690 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:17.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.935395 22690 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:17.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.936086 22690 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.967097 22690 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:13:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.967331 22690 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.967583 22690 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.967774 22690 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.967958 22690 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.968142 22690 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:17.968317 22690 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39721/jwks.json
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:18.003881 22690 master_runner.cc:386] Master server non-default flags:
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.184.84:43269
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.184.126:39787,127.21.184.125:44107,127.21.184.124:36891
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/krb5kdc/kudu_127.21.184.126.keytab
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.21.184.126
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.184.126:39787
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39721/jwks.json
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/info.pb
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.184.126
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41921
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/logs
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:18.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
14:13:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 13:43:20 UTC on 1a15035a6830
14:13:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 92
14:13:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:18.005111 22690 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:18.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:18.006469 22690 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:18.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:18.023445 22700 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:18.800859 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:18.801119 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:18.024104 22703 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:19.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:18.025677 22690 server_base.cc:1034] running on GCE node
14:13:19.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:18.025094 22701 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.172277 22690 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.174971 22690 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.176384 22690 hybrid_clock.cc:648] HybridClock initialized: now 1714659199176338 us; error 58 us; skew 500 ppm
14:13:19.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.177248 22690 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:19 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659199, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:19.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.183481 22690 init.cc:377] Logged in from keytab as kudu/127.21.184.126@KRBTEST.COM (short username kudu)
14:13:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.189535 22690 webserver.cc:460] Webserver started at http://127.21.184.126:41921/ using document root <none> and password file <none>
14:13:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.190505 22690 fs_manager.cc:359] Metadata directory not provided
14:13:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.190744 22690 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:19.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.198810 22690 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
14:13:19.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.203094 22711 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.204177 22690 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:13:19.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.204553 22690 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data,/tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79797cf15ca24cc58d04b4e03ffcdcff"
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 14:13:09 on dist-test-slave-smf8"
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.206928 22690 fs_report.cc:383] FS layout report
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/data
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:19.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.271070 22690 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.277119 22690 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.277647 22690 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.350555 22690 rpc_server.cc:307] RPC server started. Bound to: 127.21.184.126:39787
14:13:19.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.350677 22762 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.184.126:39787 every 8 connection(s)
14:13:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.353305 22690 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/data/info.pb
14:13:19.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.357743 22241 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu as pid 22690
14:13:19.358 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.364202 22763 sys_catalog.cc:263] Verifying existing consensus state
14:13:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 14:13:19.364964 22763 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:13:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.365370 22690 master.cc:537] Master@127.21.184.126:39787 shutting down...
14:13:19.410 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.184.124:36891(127.21.184.124:36891)] connecting to peer
14:13:19.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:19.468150 22690 master.cc:552] Master@127.21.184.126:39787 shutdown complete.
14:13:19.473 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:19.476 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:19.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQ4kZtv/test-tmp/mini-kudu-cluster8881325994519379056/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:13:19.511 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.21.184.124:36891(127.21.184.124:36891)] Successfully connected to peer
14:13:19.513 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.517 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.584 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:13:19.601 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.21.184.124
14:13:19.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:19 dist-test-slave-smf8 krb5kdc[22253](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659193, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.21.184.124@KRBTEST.COM
14:13:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:19.804912 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:19.805265 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:19.810 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa8285938, L:/127.0.0.1:41700 - R:/127.21.184.124:36891] using SASL/GSSAPI
14:13:19.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:19.845 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:19.846 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:19.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.851 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:19.863 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:19.864 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:19.866 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.866 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.867 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.867 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.869 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:19.883 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:19.884 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:19.886 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.887 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.888 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.888 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.889 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:19.902 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:19.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:19.907 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.907 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.907 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.908 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.908 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.909 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:19.923 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:19.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:19.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.927 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.928 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.928 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.928 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.929 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:19.942 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:19.944 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:19.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.946 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.948 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:19.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:19.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:20.062 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:20.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.065 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:20.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.068 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.068 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.068 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.069 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:20.083 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:20.084 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:20.084 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.088 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.088 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.088 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.089 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:20.403 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:20.408 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:20.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.410 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:20.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:20.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:20.773200 22770 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:20.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:20.773556 22770 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:20.774920 22770 leader_election.cc:288] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787), 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107)
14:13:20.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:20.777819 22547 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111) [suppressed 3 similar messages]
14:13:20.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:20.780761 22549 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787): Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111)
14:13:20.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:20.781189 22547 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107): Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:20.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:20.781471 22547 leader_election.cc:302] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2bc78ebecb52427d96121b13b64f3c52; no voters: 0824e38f238d4f93b0fb9d1eb74d0520, 79797cf15ca24cc58d04b4e03ffcdcff
14:13:20.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:20.782011 22770 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:13:20.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:20.802968 22527 master_runner.cc:186] Error getting master registration for 127.21.184.126:39787: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111),
14:13:20.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:20.803359 22527 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:20.807883 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:20.808136 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:21.143 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:21.144 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:21.145 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:21.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:21.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:21.148 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:21.148 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:21.148 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:21.148 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:21.324 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:21.325 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:21.326 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:21.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:21.327 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:21.327 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:21.328 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:21.328 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:21.328 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:21.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:21.810848 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:21.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:21.811110 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:22.693744 22774 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:22.694195 22774 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:22.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:22.696014 22774 leader_election.cc:288] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787), 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107)
14:13:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:22.705289 22547 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107): Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:22.706568 22549 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787): Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111)
14:13:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:22.707263 22549 leader_election.cc:302] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2bc78ebecb52427d96121b13b64f3c52; no voters: 0824e38f238d4f93b0fb9d1eb74d0520, 79797cf15ca24cc58d04b4e03ffcdcff
14:13:22.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:22.708328 22774 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:13:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:22.813899 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:22.814177 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:23.123 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:23.124 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:23.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:23.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:23.126 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:23.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:23.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:23.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:23.128 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:23.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:23.817126 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:23.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:23.817397 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:24.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:24.363816 21590 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4380 after lost signal to thread 21527
14:13:24.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:24.364831 21590 debug-util.cc:398] Leaking SignalData structure 0x7b080009d240 after lost signal to thread 21593
14:13:24.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:24.418139 22782 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:24.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:24.418740 22782 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:24.420969 22782 leader_election.cc:288] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787), 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107)
14:13:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:24.426592 22549 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787): Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111)
14:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:24.427699 22547 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107): Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:24.428105 22547 leader_election.cc:302] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2bc78ebecb52427d96121b13b64f3c52; no voters: 0824e38f238d4f93b0fb9d1eb74d0520, 79797cf15ca24cc58d04b4e03ffcdcff
14:13:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:24.428920 22782 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:13:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:24.820662 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:24.820976 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:25.003 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:25.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:25.006 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:25.007 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:25.007 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:25.007 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:25.008 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:25.008 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:25.009 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:25 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659205, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:25.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:25.480916 22542 init.cc:295] Successfully reacquired a new kerberos TGT
14:13:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:25.806489 22527 master_runner.cc:186] Error getting master registration for 127.21.184.126:39787: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111),
14:13:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:25.807080 22527 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:25.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:25.823695 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:25.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:25.823938 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:26.826635 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:26.826889 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:27.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:27.829552 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:27.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:27.829810 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:28.483 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:28.484 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:28.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:28.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:28.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:28.487 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:28.488 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:28.488 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:28.488 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:28.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:28.832286 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:28.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:28.832530 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:29.835016 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:29.835270 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:30.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:30.809746 22527 master_runner.cc:186] Error getting master registration for 127.21.184.126:39787: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111),
14:13:30.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:30.810096 22527 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:30.838244 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:30.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:30.838502 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:31.342 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:31.344 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:31.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:31.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:31.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:31.347 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:31.347 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:31.347 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:31.347 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:31.841171 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:31.841439 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:32.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:32.844244 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:32.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:32.844532 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:32.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:32.975586 22796 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:32.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:32.975960 22796 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:32.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:32.977517 22796 leader_election.cc:288] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787), 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107)
14:13:32.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:32.979375 22549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111) [suppressed 5 similar messages]
14:13:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:32.982810 22549 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787): Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111)
14:13:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:32.983165 22547 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107): Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:32.983449 22547 leader_election.cc:302] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2bc78ebecb52427d96121b13b64f3c52; no voters: 0824e38f238d4f93b0fb9d1eb74d0520, 79797cf15ca24cc58d04b4e03ffcdcff
14:13:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:32.983975 22796 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:13:33.543 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:33.544 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:33.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:33.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:33.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:33.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:33.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:33.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:33.547 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:33.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:33.847307 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:33.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:33.847591 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:34.464 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:34.465 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:34.466 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:34.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:34.467 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:34.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:34.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:34.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:34.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:34.850767 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:34.851099 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:35.023 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:35.024 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:35.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:35.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:35.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:35.027 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:35.027 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:35.027 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:35.027 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:35.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:35 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659215, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:35.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:35.483754 22542 init.cc:295] Successfully reacquired a new kerberos TGT
14:13:35.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:35.813463 22527 master_runner.cc:186] Error getting master registration for 127.21.184.126:39787: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111),
14:13:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:35.813836 22527 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:35.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:35.853888 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:35.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:35.854146 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:36.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:36.856889 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:36.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:36.857146 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:37.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:37.859689 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:37.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:37.859972 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:38.083 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:38.085 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:38.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:38.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:38.086 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:38.088 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:38.088 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:38.088 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:38.089 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:38.862614 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:38.862869 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:38.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:38.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:38.904 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:38.904 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:38.904 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:38.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:38.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:38.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:38.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:39.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:39.865417 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:39.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:39.865813 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:40.143 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:40.144 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:40.144 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:40.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:40.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:40.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:40.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:40.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:40.146 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:40.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:40.817328 22527 master_runner.cc:186] Error getting master registration for 127.21.184.126:39787: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111),
14:13:40.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:40.817682 22527 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:40.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:40.868733 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:40.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:40.868997 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:41.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:41.871562 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:41.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:41.871814 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:42.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:42.874436 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:42.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:42.874696 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:43.343 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:43.343 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:43.344 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:43.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:43.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:43.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:43.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:43.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:43.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:43.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:43.877456 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:43.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:43.877712 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:44.881206 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:44.881522 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:45.063 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:45.063 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:45.064 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:45.064 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:45.064 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:45.064 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:45.064 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:45.065 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:45.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 14:13:45 dist-test-slave-smf8 krb5kdc[22253](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714659225, etypes {rep=17 tkt=17 ses=17}, kudu/127.21.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:13:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:45.486091 22542 init.cc:295] Successfully reacquired a new kerberos TGT
14:13:45.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:45.820855 22527 master_runner.cc:186] Error getting master registration for 127.21.184.126:39787: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111),
14:13:45.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:45.821234 22527 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:45.884133 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:45.884370 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:46.795917 22820 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:46.796437 22820 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79797cf15ca24cc58d04b4e03ffcdcff" member_type: VOTER last_known_addr { host: "127.21.184.126" port: 39787 } } peers { permanent_uuid: "0824e38f238d4f93b0fb9d1eb74d0520" member_type: VOTER last_known_addr { host: "127.21.184.125" port: 44107 } } peers { permanent_uuid: "2bc78ebecb52427d96121b13b64f3c52" member_type: VOTER last_known_addr { host: "127.21.184.124" port: 36891 } }
14:13:46.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:46.798094 22820 leader_election.cc:288] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787), 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107)
14:13:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:46.800307 22549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:13:46.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:46.803678 22547 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0824e38f238d4f93b0fb9d1eb74d0520 (127.21.184.125:44107): Network error: Client connection negotiation failed: client connection to 127.21.184.125:44107: connect: Connection refused (error 111)
14:13:46.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:46.804172 22549 leader_election.cc:334] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79797cf15ca24cc58d04b4e03ffcdcff (127.21.184.126:39787): Network error: Client connection negotiation failed: client connection to 127.21.184.126:39787: connect: Connection refused (error 111)
14:13:46.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:46.804482 22549 leader_election.cc:302] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2bc78ebecb52427d96121b13b64f3c52; no voters: 0824e38f238d4f93b0fb9d1eb74d0520, 79797cf15ca24cc58d04b4e03ffcdcff
14:13:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:46.805132 22820 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:13:46.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:46.886658 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:46.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:46.886910 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:46.942 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:46.943 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:46.944 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:46.944 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:46.944 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:46.944 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:46.944 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:46.944 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:46.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:47.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:47.468801 21665 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1140 after lost signal to thread 21600
14:13:47.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:47.469364 21665 debug-util.cc:398] Leaking SignalData structure 0x7b08000aef00 after lost signal to thread 21668
14:13:47.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:47.889434 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:47.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:47.889668 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:48.892038 22615 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2bc78ebecb52427d96121b13b64f3c52: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 14:13:48.892308 22615 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:49.343 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.125:44107(127.21.184.125:44107)] connecting to peer
14:13:49.343 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.21.184.126:39787(127.21.184.126:39787)] connecting to peer
14:13:49.344 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:49.344 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.21.184.125:44107(127.21.184.125:44107)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:49.344 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:49.344 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.125:44107: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107
14:13:49.344 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.126:39787(127.21.184.126:39787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:49.345 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.21.184.126:39787: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
14:13:49.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787; will retry
14:13:49.350 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29985), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [204ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [457ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [461ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [477ms] refreshing cache from master, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [485ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [487ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [503ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [522ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [523ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [525ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [542ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [544ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [561ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [562ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [564ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [581ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [586ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [700ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [702ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [703ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [704ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [720ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [722ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [724ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1037ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1045ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1047ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1777ms] refreshing cache from master, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1783ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1957ms] refreshing cache from master, [1958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1963ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1963ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [3757ms] refreshing cache from master, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [3759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [3761ms] too many traces: truncated at 100 traces, deferred=Deferred@1542213246(state=PENDING, result=null, callback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399), errback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399)))
14:13:49.354 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.21.184.124:36891(127.21.184.124:36891)] disconnecting while in state READY
14:13:49.355 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.21.184.125:44107(127.21.184.125:44107)] disconnecting while in state TERMINATED
14:13:49.356 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.21.184.126:39787(127.21.184.126:39787)] disconnecting while in state TERMINATED
14:13:49.359 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:13:49.361 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.21.184.124:36891(127.21.184.124:36891)] handling channelInactive
14:13:49.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.21.184.124:36891(127.21.184.124:36891)] cleaning up while in state READY due to: connection closed
14:13:49.362 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:13:49.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:49.365644 22241 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 22527
14:13:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:13:50Z chronyd exiting
14:13:51.458 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:13:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:51.459656 21514 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 21796
14:13:51.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:51.484395 21514 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 21955
14:13:51.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:51.511011 21514 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 22099
14:13:51.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:51.539125 21514 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 21526
14:13:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:51.569756 21514 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 21596
14:13:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 14:13:51.601399 21514 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQ4kZtv/build/tsan/bin/kudu with pid 21671
14:13:52.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T14:13:52Z chronyd exiting
14:13:53.653 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29985), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [204ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [457ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [461ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [477ms] refreshing cache from master, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [485ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [487ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [503ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [522ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [523ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [525ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [542ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [544ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [561ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [562ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [564ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [581ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [586ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [700ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [702ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [703ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [704ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [720ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [722ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [724ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1037ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1045ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1047ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1777ms] refreshing cache from master, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1783ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1957ms] refreshing cache from master, [1958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1963ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1963ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [3757ms] refreshing cache from master, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [3759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [3761ms] too many traces: truncated at 100 traces, deferred=Deferred@1542213246(state=PENDING, result=null, callback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399), errback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.21.184.126:39787
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29985), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [204ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [457ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [461ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [477ms] refreshing cache from master, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [485ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [487ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [503ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [522ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [523ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [525ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [542ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [544ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [561ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [562ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [564ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [581ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [586ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [700ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [702ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [703ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [704ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [720ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [722ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [724ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1037ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1045ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1047ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1777ms] refreshing cache from master, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1783ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1957ms] refreshing cache from master, [1958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1963ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1963ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [3757ms] refreshing cache from master, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [3759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [3761ms] too many traces: truncated at 100 traces, deferred=Deferred@1542213246(state=PENDING, result=null, callback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399), errback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.21.184.126:39787
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14:13:54.041 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
14:13:54.041 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29985), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [204ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [457ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [461ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [477ms] refreshing cache from master, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [485ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [487ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [503ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [522ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [523ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [525ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [542ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [544ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [561ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [562ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [564ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [581ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [586ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [700ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [702ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [703ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [704ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [720ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [722ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [724ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1037ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1045ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1047ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1777ms] refreshing cache from master, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1783ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1957ms] refreshing cache from master, [1958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1963ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1963ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [3757ms] refreshing cache from master, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [3759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [3761ms] too many traces: truncated at 100 traces, deferred=Deferred@1542213246(state=PENDING, result=null, callback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399), errback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.21.184.126:39787
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29985), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [153ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [204ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [457ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [461ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [477ms] refreshing cache from master, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [484ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [485ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [487ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [497ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [502ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [503ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [504ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [522ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [523ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [525ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [542ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [544ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [561ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [562ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [564ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [581ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [586ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [700ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [702ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [703ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [704ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [720ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [722ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [724ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1037ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1045ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1047ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1777ms] refreshing cache from master, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1783ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1957ms] refreshing cache from master, [1958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [1959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [1960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.125:44107: Network error: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107, [1962ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.124:36891: OK, [1963ms] Sub RPC ConnectToMaster: received response from server master-127.21.184.126:39787: Network error: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [1963ms] delaying RPC due to: Service unavailable: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787, [3757ms] refreshing cache from master, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.124:36891, [3758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.125:44107, [3759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.21.184.126:39787, [3761ms] too many traces: truncated at 100 traces, deferred=Deferred@1542213246(state=PENDING, result=null, callback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399), errback=(continuation of Deferred@1277396869 after retry RPC after error@927520104) -> (continuation of Deferred@1498086341 after retry RPC after error@1626132780) -> (continuation of Deferred@807628059 after retry RPC after error@1102122466) -> (continuation of Deferred@1328239298 after retry RPC after error@1229496250) -> (continuation of Deferred@1655207997 after retry RPC after error@725596055) -> (continuation of Deferred@1041714544 after retry RPC after error@858759239) -> (continuation of Deferred@308764559 after retry RPC after error@1144181083) -> (continuation of Deferred@1078622304 after retry RPC after error@237622710) -> (continuation of Deferred@647657691 after retry RPC after error@1953553754) -> (continuation of Deferred@1735013068 after retry RPC after error@657383410) -> (continuation of Deferred@1376204652 after retry RPC after error@552581755) -> (continuation of Deferred@371523832 after retry RPC after error@1193643137) -> (continuation of Deferred@745697118 after retry RPC after error@1905468617) -> (continuation of Deferred@1326434499 after retry RPC after error@606409767) -> (continuation of Deferred@454203234 after retry RPC after error@1934581157) -> (continuation of Deferred@1119341279 after retry RPC after error@1321171586) -> (continuation of Deferred@1092181424 after retry RPC after error@1219658307) -> (continuation of Deferred@875518136 after retry RPC after error@32595925) -> (continuation of Deferred@177104518 after retry RPC after error@1458724110) -> (continuation of Deferred@408303088 after retry RPC after error@324198104) -> (continuation of Deferred@1787895906 after retry RPC after error@785320467) -> (continuation of Deferred@119493519 after retry RPC after error@46070186) -> (continuation of Deferred@1632586671 after retry RPC after error@923163143) -> (continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.21.184.124:36891,127.21.184.125:44107,127.21.184.126:39787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.125:44107(127.21.184.125:44107): Connection refused: /127.21.184.125:44107,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.21.184.126:39787(127.21.184.126:39787): Connection refused: /127.21.184.126:39787
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.21.184.126:39787
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)