08:21:23.165 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:21:23.178 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskcW3gy0/test-tmp
08:21:23.180 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:21:23.184 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskcW3gy0/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:21:23.391 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:21:23.601 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:21:23.603 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:21:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:23.607326 26532 mini_oidc.cc:152] Starting JWKS server
08:21:23.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:23.611340 26532 webserver.cc:460] Webserver started at http://127.0.0.1:46797/ using document root <none> and password file <none>
08:21:23.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:23.611864 26532 mini_oidc.cc:189] Starting OIDC Discovery server
08:21:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:23.613896 26532 webserver.cc:460] Webserver started at http://127.0.0.1:40243/ using document root <none> and password file <none>
08:21:23.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T08:21:23Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:21:23.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T08:21:23Z Disabled control of system clock
08:21:23.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:23.682446 26532 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:23.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:23.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/wal
08:21:23.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/data
08:21:23.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:23.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:23.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:23.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:23.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:23.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:23.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/logs
08:21:23.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/data/info.pb
08:21:23.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.62:33477
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.62
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46797/jwks.json with env {}
08:21:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:23.993515 26544 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:23.994122 26544 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:23.994784 26544 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:24.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.028954 26544 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:24.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.029289 26544 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:24.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.029495 26544 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:24.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.029683 26544 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:24.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.029947 26544 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:24.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.030176 26544 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46797/jwks.json
08:21:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:24.067157 26544 master_runner.cc:386] Master server non-default flags:
08:21:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/data
08:21:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/wal
08:21:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.62:33477
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46797/jwks.json
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/data/info.pb
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.62
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/logs
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:24.068617 26544 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:24.071007 26544 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.091168 26550 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:25.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.091928 26553 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:24.093897 26544 server_base.cc:1034] running on GCE node
08:21:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:24.092746 26551 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:25.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.213464 26544 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:25.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.217029 26544 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.218571 26544 hybrid_clock.cc:648] HybridClock initialized: now 1714638085218502 us; error 89 us; skew 500 ppm
08:21:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.219537 26544 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:25.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.231369 26544 webserver.cc:460] Webserver started at http://127.25.233.62:41807/ using document root <none> and password file <none>
08:21:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.232419 26544 fs_manager.cc:359] Metadata directory not provided
08:21:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.232630 26544 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:25.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.233050 26544 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:25.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.238255 26544 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/data/instance:
08:21:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5"
08:21:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:25 on dist-test-slave-rhmj"
08:21:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.239497 26544 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/wal/instance:
08:21:25.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5"
08:21:25.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:25 on dist-test-slave-rhmj"
08:21:25.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.247516 26544 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
08:21:25.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.254156 26560 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:25.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.255445 26544 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
08:21:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.255887 26544 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/wal
08:21:25.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5"
08:21:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:25 on dist-test-slave-rhmj"
08:21:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.256340 26544 fs_report.cc:383] FS layout report
08:21:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/wal
08:21:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/wal
08:21:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/data/data
08:21:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:25.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.313238 26544 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:25.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.314975 26544 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:25.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.315531 26544 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.389472 26544 rpc_server.cc:307] RPC server started. Bound to: 127.25.233.62:33477
08:21:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.389588 26612 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.233.62:33477 every 8 connection(s)
08:21:25.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.392565 26544 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/data/info.pb
08:21:25.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.393949 26532 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 26544
08:21:25.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.394421 26532 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-0/wal/instance
08:21:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.398855 26532 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/wal
08:21:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/data
08:21:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/logs
08:21:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/data/info.pb
08:21:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.61:36305
08:21:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.61
08:21:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46797/jwks.json with env {}
08:21:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.399398 26613 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:21:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.414929 26613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } has no permanent_uuid. Determining permanent_uuid...
08:21:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.443333 26613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } has no permanent_uuid. Determining permanent_uuid...
08:21:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.445709 26562 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.233.61:36305: connect: Connection refused (error 111)
08:21:25.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.450058 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.61:36305: Network error: Client connection negotiation failed: client connection to 127.25.233.61:36305: connect: Connection refused (error 111)
08:21:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.499751 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } attempt: 1
08:21:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.504462 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.61:36305: Network error: Client connection negotiation failed: client connection to 127.25.233.61:36305: connect: Connection refused (error 111)
08:21:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.573091 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } attempt: 2
08:21:25.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.577411 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.61:36305: Network error: Client connection negotiation failed: client connection to 127.25.233.61:36305: connect: Connection refused (error 111)
08:21:25.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.668900 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } attempt: 3
08:21:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.673063 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.61:36305: Network error: Client connection negotiation failed: client connection to 127.25.233.61:36305: connect: Connection refused (error 111)
08:21:25.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.725832 26615 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:25.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.726403 26615 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.726987 26615 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.757859 26615 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.758157 26615 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.758370 26615 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.758616 26615 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.758836 26615 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.758985 26615 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46797/jwks.json
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.793669 26615 master_runner.cc:386] Master server non-default flags:
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/data
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/wal
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:25.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.61:36305
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46797/jwks.json
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/data/info.pb
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.61
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:25.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/logs
08:21:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.794983 26615 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.797211 26615 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:25.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.814945 26624 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:25.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.816510 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } attempt: 4
08:21:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.820421 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.61:36305: Network error: Client connection negotiation failed: client connection to 127.25.233.61:36305: connect: Connection refused (error 111)
08:21:26.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.119902 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } attempt: 5
08:21:26.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:26.123960 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.61:36305: Network error: Client connection negotiation failed: client connection to 127.25.233.61:36305: connect: Connection refused (error 111)
08:21:26.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.671413 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } attempt: 6
08:21:26.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:26.678745 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.61:36305: Network error: Client connection negotiation failed: client connection to 127.25.233.61:36305: connect: Connection refused (error 111)
08:21:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.815310 26625 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:26.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:25.817410 26615 server_base.cc:1034] running on GCE node
08:21:26.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:25.816689 26627 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.926803 26615 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:26.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.929620 26615 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.931015 26615 hybrid_clock.cc:648] HybridClock initialized: now 1714638086930940 us; error 80 us; skew 500 ppm
08:21:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.932116 26615 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:26.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.938334 26615 webserver.cc:460] Webserver started at http://127.25.233.61:39993/ using document root <none> and password file <none>
08:21:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.939306 26615 fs_manager.cc:359] Metadata directory not provided
08:21:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.939522 26615 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.939986 26615 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:26.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.945040 26615 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/data/instance:
08:21:26.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8237c8b87b24b54a954b3e7e50414ff"
08:21:26.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:26 on dist-test-slave-rhmj"
08:21:26.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.946151 26615 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/wal/instance:
08:21:26.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8237c8b87b24b54a954b3e7e50414ff"
08:21:26.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:26 on dist-test-slave-rhmj"
08:21:26.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.953488 26615 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
08:21:26.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.959117 26636 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.960316 26615 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:21:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.960641 26615 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/wal
08:21:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8237c8b87b24b54a954b3e7e50414ff"
08:21:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:26 on dist-test-slave-rhmj"
08:21:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:26.961043 26615 fs_report.cc:383] FS layout report
08:21:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/wal
08:21:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/wal
08:21:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/data/data
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.010861 26615 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.013206 26615 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.013880 26615 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:27.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.086817 26615 rpc_server.cc:307] RPC server started. Bound to: 127.25.233.61:36305
08:21:27.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.086934 26687 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.233.61:36305 every 8 connection(s)
08:21:27.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.089578 26615 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/data/info.pb
08:21:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.091413 26532 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 26615
08:21:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.091975 26532 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-1/wal/instance
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.094735 26532 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/wal
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/data
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/logs
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/data/info.pb
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.60:38453
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.60
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46797/jwks.json with env {}
08:21:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.096940 26688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:21:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.110088 26688 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } has no permanent_uuid. Determining permanent_uuid...
08:21:27.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.131827 26688 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } has no permanent_uuid. Determining permanent_uuid...
08:21:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.147312 26688 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } has no permanent_uuid. Determining permanent_uuid...
08:21:27.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.149995 26640 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.154182 26688 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.203743 26688 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 1
08:21:27.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.208386 26688 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.276809 26688 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 2
08:21:27.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.281414 26688 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.372915 26688 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 3
08:21:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.376825 26688 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.413919 26690 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:27.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.414472 26690 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:27.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.415110 26690 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.446673 26690 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.446965 26690 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.447160 26690 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.447342 26690 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.447551 26690 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:27.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.447721 26690 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46797/jwks.json
08:21:27.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.483147 26690 master_runner.cc:386] Master server non-default flags:
08:21:27.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/data
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/wal
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.60:38453
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46797/jwks.json
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/data/info.pb
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.60
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/logs
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.484498 26690 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.486392 26690 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.504652 26701 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:27.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.520282 26688 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 4
08:21:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.525380 26688 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.739486 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } attempt: 7
08:21:27.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.753084 26613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } has no permanent_uuid. Determining permanent_uuid...
08:21:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.758773 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.817545 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 1
08:21:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.822655 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.824872 26688 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 5
08:21:27.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.829704 26688 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.904063 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 2
08:21:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.908788 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.994258 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 3
08:21:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.998703 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:28.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.139423 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 4
08:21:28.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:28.143883 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.377144 26688 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 6
08:21:28.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:28.387167 26688 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:28.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.427379 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 5
08:21:28.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:28.432634 26613 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.25.233.60:38453: Network error: Client connection negotiation failed: client connection to 127.25.233.60:38453: connect: Connection refused (error 111)
08:21:28.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.505923 26700 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:28.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:27.509738 26690 server_base.cc:1034] running on GCE node
08:21:28.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:27.508738 26703 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:28.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.669523 26690 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:28.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.673074 26690 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:28.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.674582 26690 hybrid_clock.cc:648] HybridClock initialized: now 1714638088674521 us; error 87 us; skew 500 ppm
08:21:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.675436 26690 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:28.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.682614 26690 webserver.cc:460] Webserver started at http://127.25.233.60:33693/ using document root <none> and password file <none>
08:21:28.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.683562 26690 fs_manager.cc:359] Metadata directory not provided
08:21:28.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.683818 26690 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:28.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.684232 26690 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.689352 26690 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/data/instance:
08:21:28.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22104fbab61e4fe08ec14352ba5a4969"
08:21:28.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:28 on dist-test-slave-rhmj"
08:21:28.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.690623 26690 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/wal/instance:
08:21:28.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22104fbab61e4fe08ec14352ba5a4969"
08:21:28.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:28 on dist-test-slave-rhmj"
08:21:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.700795 26690 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.005s
08:21:28.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.708427 26716 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:28.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.709616 26690 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
08:21:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.709932 26690 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/wal
08:21:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22104fbab61e4fe08ec14352ba5a4969"
08:21:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:28 on dist-test-slave-rhmj"
08:21:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.710253 26690 fs_report.cc:383] FS layout report
08:21:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/wal
08:21:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/wal
08:21:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/data/data
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:28.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:28.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.761749 26690 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:28.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.763306 26690 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:28.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.763808 26690 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:28.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.839274 26690 rpc_server.cc:307] RPC server started. Bound to: 127.25.233.60:38453
08:21:28.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.839447 26767 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.233.60:38453 every 8 connection(s)
08:21:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.841984 26690 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/data/info.pb
08:21:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.844969 26532 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 26690
08:21:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.845491 26532 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/master-2/wal/instance
08:21:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.849066 26768 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:21:28.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.869469 26768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } has no permanent_uuid. Determining permanent_uuid...
08:21:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.899899 26768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } has no permanent_uuid. Determining permanent_uuid...
08:21:28.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.910420 26768 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } has no permanent_uuid. Determining permanent_uuid...
08:21:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.928324 26768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: Bootstrap starting.
08:21:28.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.935333 26768 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: Neither blocks nor log segments found. Creating new log.
08:21:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.937345 26768 log.cc:824] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: Log is configured to *not* fsync() on all Append() calls
08:21:28.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.942414 26768 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: No bootstrap required, opened a new log
08:21:28.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.965299 26768 raft_consensus.cc:348] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:28.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.966040 26768 raft_consensus.cc:374] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:21:28.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.966245 26768 raft_consensus.cc:729] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22104fbab61e4fe08ec14352ba5a4969, State: Initialized, Role: FOLLOWER
08:21:28.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.967072 26768 consensus_queue.cc:260] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:28.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.970171 26777 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:28.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.970882 26777 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [sys.catalog]: This master's current role is: FOLLOWER
08:21:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.972376 26768 sys_catalog.cc:564] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [sys.catalog]: configured and running, proceeding with master startup.
08:21:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.985085 26613 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 6
08:21:29.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:29.006242 26789 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:29.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:29.006716 26789 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:29.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:29.019982 26722 tablet.cc:2369] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:21:29.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.025934 26613 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5: Bootstrap starting.
08:21:29.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.031369 26613 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5: Neither blocks nor log segments found. Creating new log.
08:21:29.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:28.999850 26690 master_runner.cc:186] Error getting master registration for 127.25.233.62:33477: OK, instance_id { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" instance_seqno: 1714638085368941 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:21:29.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.032676 26690 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:21:29.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.033979 26613 log.cc:824] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5: Log is configured to *not* fsync() on all Append() calls
08:21:29.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.040591 26613 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5: No bootstrap required, opened a new log
08:21:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.065079 26613 raft_consensus.cc:348] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.065913 26613 raft_consensus.cc:374] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:21:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.066205 26613 raft_consensus.cc:729] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6d46fa77a9e4fa1ab2f0efbcdfca5b5, State: Initialized, Role: FOLLOWER
08:21:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.067180 26613 consensus_queue.cc:260] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:29.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.070498 26792 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:29.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.071465 26792 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: This master's current role is: FOLLOWER
08:21:29.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.072505 26613 sys_catalog.cc:564] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: configured and running, proceeding with master startup.
08:21:29.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:29.075371 26566 tablet.cc:2369] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:21:29.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:29.133949 26803 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:29.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:29.134894 26803 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.135787 26792 raft_consensus.cc:482] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:21:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.136327 26792 raft_consensus.cc:504] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:29.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.141708 26792 leader_election.cc:288] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f8237c8b87b24b54a954b3e7e50414ff (127.25.233.61:36305), 22104fbab61e4fe08ec14352ba5a4969 (127.25.233.60:38453)
08:21:29.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.142232 26743 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22104fbab61e4fe08ec14352ba5a4969" is_pre_election: true
08:21:29.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.143335 26743 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6d46fa77a9e4fa1ab2f0efbcdfca5b5 in term 0.
08:21:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.145058 26564 leader_election.cc:302] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 22104fbab61e4fe08ec14352ba5a4969, c6d46fa77a9e4fa1ab2f0efbcdfca5b5; no voters:
08:21:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.146790 26792 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:21:29.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.147756 26792 raft_consensus.cc:482] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:21:29.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.148684 26792 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 0 FOLLOWER]: Advancing to term 1
08:21:29.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.162133 26792 raft_consensus.cc:504] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:29.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.116765 26544 master_runner.cc:186] Error getting master registration for 127.25.233.61:36305: OK, instance_id { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" instance_seqno: 1714638087066825 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:21:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.166211 26544 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:21:29.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.170157 26792 leader_election.cc:288] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [CANDIDATE]: Term 1 election: Requested vote from peers f8237c8b87b24b54a954b3e7e50414ff (127.25.233.61:36305), 22104fbab61e4fe08ec14352ba5a4969 (127.25.233.60:38453)
08:21:29.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.172088 26743 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22104fbab61e4fe08ec14352ba5a4969"
08:21:29.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.172732 26743 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [term 0 FOLLOWER]: Advancing to term 1
08:21:29.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.188036 26743 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6d46fa77a9e4fa1ab2f0efbcdfca5b5 in term 1.
08:21:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.190506 26564 leader_election.cc:302] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 22104fbab61e4fe08ec14352ba5a4969, c6d46fa77a9e4fa1ab2f0efbcdfca5b5; no voters:
08:21:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.192065 26792 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 1 FOLLOWER]: Leader election won for term 1
08:21:29.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.210052 26792 raft_consensus.cc:686] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [term 1 LEADER]: Becoming Leader. State: Replica: c6d46fa77a9e4fa1ab2f0efbcdfca5b5, State: Running, Role: LEADER
08:21:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.213404 26792 consensus_queue.cc:237] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.230022 26806 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c6d46fa77a9e4fa1ab2f0efbcdfca5b5. Latest consensus state: current_term: 1 leader_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.231081 26806 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: This master's current role is: LEADER
08:21:29.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.235380 26808 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
08:21:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.243023 26808 catalog_manager.cc:1484] Initializing Kudu cluster ID...
08:21:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.275550 26743 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [term 1 FOLLOWER]: Refusing update from remote peer c6d46fa77a9e4fa1ab2f0efbcdfca5b5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:21:29.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.282543 26792 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
08:21:29.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.313411 26809 mvcc.cc:205] Tried to move back new op lower bound from 7023157613654577152 to 7023157613446569984. Current Snapshot: MvccSnapshot[applied={T|T < 7023157613654577152}]
08:21:29.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.318746 26777 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c6d46fa77a9e4fa1ab2f0efbcdfca5b5. Latest consensus state: current_term: 1 leader_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.322520 26777 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [sys.catalog]: This master's current role is: FOLLOWER
08:21:29.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.327337 26806 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:29.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.328161 26806 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: This master's current role is: LEADER
08:21:29.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.329782 26813 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:29.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.330637 26813 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969 [sys.catalog]: This master's current role is: FOLLOWER
08:21:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.334954 26808 catalog_manager.cc:1347] Generated new cluster ID: d06e3d2a59dc482f8c02ca222e33c202
08:21:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.335268 26808 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
08:21:29.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.420644 26808 catalog_manager.cc:1370] Generated new certificate authority record
08:21:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.422811 26808 catalog_manager.cc:1504] Loading token signing keys...
08:21:29.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.446959 26808 catalog_manager.cc:5857] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5: Generated new TSK 0
08:21:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.447566 26688 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } attempt: 7
08:21:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.448218 26808 catalog_manager.cc:1514] Initializing in-progress tserver states...
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.465463 26532 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/wal
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/data
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/logs
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/data/info.pb
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.1:0
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.233.1
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.1
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:21:29.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.487941 26688 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff: Bootstrap starting.
08:21:29.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.512042 26688 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff: Neither blocks nor log segments found. Creating new log.
08:21:29.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.515154 26688 log.cc:824] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff: Log is configured to *not* fsync() on all Append() calls
08:21:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.521157 26688 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff: No bootstrap required, opened a new log
08:21:29.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.595526 26688 raft_consensus.cc:348] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:29.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.597884 26688 raft_consensus.cc:374] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:21:29.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.598654 26688 raft_consensus.cc:729] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f8237c8b87b24b54a954b3e7e50414ff, State: Initialized, Role: FOLLOWER
08:21:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.601060 26688 consensus_queue.cc:260] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } }
08:21:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.616797 26818 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.617748 26818 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [sys.catalog]: This master's current role is: FOLLOWER
08:21:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.624367 26662 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f8237c8b87b24b54a954b3e7e50414ff"
08:21:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.625309 26662 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [term 0 FOLLOWER]: Advancing to term 1
08:21:29.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.628135 26663 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f8237c8b87b24b54a954b3e7e50414ff" is_pre_election: true
08:21:29.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.635739 26688 sys_catalog.cc:564] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [sys.catalog]: configured and running, proceeding with master startup.
08:21:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.637151 26662 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6d46fa77a9e4fa1ab2f0efbcdfca5b5 in term 1.
08:21:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.638825 26661 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [term 1 FOLLOWER]: Refusing update from remote peer c6d46fa77a9e4fa1ab2f0efbcdfca5b5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:21:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.647356 26792 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
08:21:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.709921 26818 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [sys.catalog]: SysCatalogTable state changed. Reason: New leader c6d46fa77a9e4fa1ab2f0efbcdfca5b5. Latest consensus state: current_term: 1 leader_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:29.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.710779 26818 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [sys.catalog]: This master's current role is: FOLLOWER
08:21:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.710497 26825 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.713336 26825 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff [sys.catalog]: This master's current role is: FOLLOWER
08:21:29.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:29.773906 26834 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:21:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:30.005537 26816 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:30.006381 26816 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:30.007256 26816 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:30.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.013410 26789 catalog_manager.cc:1259] Loaded cluster ID: d06e3d2a59dc482f8c02ca222e33c202
08:21:30.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.014122 26789 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: loading cluster ID for follower catalog manager: success
08:21:30.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.026032 26789 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: acquiring CA information for follower catalog manager: success
08:21:30.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.032604 26789 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 22104fbab61e4fe08ec14352ba5a4969: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:21:30.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:30.070050 26816 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:30.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:30.071494 26816 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.233.1
08:21:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.136147 26816 tablet_server_runner.cc:78] Tablet server non-default flags:
08:21:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/data
08:21:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/wal
08:21:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.1:0
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/data/info.pb
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.1
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.233.1
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/logs
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:30.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:30.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.137876 26816 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:30.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.140240 26816 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:30.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:30.171460 26840 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:30.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.258987 26844 sys_catalog.cc:455] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6d46fa77a9e4fa1ab2f0efbcdfca5b5" member_type: VOTER last_known_addr { host: "127.25.233.62" port: 33477 } } peers { permanent_uuid: "f8237c8b87b24b54a954b3e7e50414ff" member_type: VOTER last_known_addr { host: "127.25.233.61" port: 36305 } } peers { permanent_uuid: "22104fbab61e4fe08ec14352ba5a4969" member_type: VOTER last_known_addr { host: "127.25.233.60" port: 38453 } } }
08:21:30.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.263397 26844 sys_catalog.cc:458] T 00000000000000000000000000000000 P c6d46fa77a9e4fa1ab2f0efbcdfca5b5 [sys.catalog]: This master's current role is: LEADER
08:21:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.808187 26834 catalog_manager.cc:1259] Loaded cluster ID: d06e3d2a59dc482f8c02ca222e33c202
08:21:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.808493 26834 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff: loading cluster ID for follower catalog manager: success
08:21:30.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.818392 26834 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff: acquiring CA information for follower catalog manager: success
08:21:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:30.839489 26834 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f8237c8b87b24b54a954b3e7e50414ff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:21:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:31.565804 26839 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 26816
08:21:31.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:31.659826 26816 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.490s user 0.504s sys 0.885s
08:21:31.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:30.173992 26841 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:31.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:31.663062 26816 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.494s user 0.504s sys 0.885s
08:21:31.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.670043 26816 server_base.cc:1034] running on GCE node
08:21:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:31.673817 26849 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.675264 26816 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:31.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.684485 26816 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.688668 26816 hybrid_clock.cc:648] HybridClock initialized: now 1714638091688619 us; error 74 us; skew 500 ppm
08:21:31.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.689896 26816 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:31.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.724398 26816 webserver.cc:460] Webserver started at http://127.25.233.1:34673/ using document root <none> and password file <none>
08:21:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.725791 26816 fs_manager.cc:359] Metadata directory not provided
08:21:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.726092 26816 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.726683 26816 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:31.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.734094 26816 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/data/instance:
08:21:31.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa030e46e2684533922e85bd94aa50ec"
08:21:31.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:31 on dist-test-slave-rhmj"
08:21:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.735710 26816 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/wal/instance:
08:21:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa030e46e2684533922e85bd94aa50ec"
08:21:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:31 on dist-test-slave-rhmj"
08:21:31.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.757889 26816 fs_manager.cc:691] Time spent creating directory manager: real 0.021s user 0.008s sys 0.012s
08:21:31.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.776692 26854 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.780804 26816 fs_manager.cc:725] Time spent opening block manager: real 0.018s user 0.005s sys 0.011s
08:21:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.781600 26816 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/wal
08:21:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa030e46e2684533922e85bd94aa50ec"
08:21:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:31 on dist-test-slave-rhmj"
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.782382 26816 fs_report.cc:383] FS layout report
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/wal
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/wal
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/data/data
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:31.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.888878 26816 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:31.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.891304 26816 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:31.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.892246 26816 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.902473 26816 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:21:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.915233 26816 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:21:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.915982 26816 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.001s
08:21:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.916564 26816 ts_tablet_manager.cc:590] Registered 0 tablets
08:21:31.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:31.917536 26816 ts_tablet_manager.cc:569] Time spent register tablets: real 0.001s user 0.000s sys 0.000s
08:21:32.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.604790 26816 rpc_server.cc:307] RPC server started. Bound to: 127.25.233.1:44721
08:21:32.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.604952 26967 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.233.1:44721 every 8 connection(s)
08:21:32.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.608594 26816 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/data/info.pb
08:21:32.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.611181 26532 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 26816
08:21:32.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.611692 26532 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-0/wal/instance
08:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.626369 26532 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/wal
08:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/data
08:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/logs
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/data/info.pb
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.2:0
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.233.2
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.2
08:21:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:21:32.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.683986 26968 heartbeater.cc:347] Connected to a master server at 127.25.233.60:38453
08:21:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.684715 26968 heartbeater.cc:464] Registering TS with master...
08:21:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.686208 26968 heartbeater.cc:511] Master 127.25.233.60:38453 requested a full tablet report, sending...
08:21:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.690827 26733 ts_manager.cc:194] Registered new tserver with Master: aa030e46e2684533922e85bd94aa50ec (127.25.233.1:44721)
08:21:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.704355 26969 heartbeater.cc:347] Connected to a master server at 127.25.233.61:36305
08:21:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.704959 26969 heartbeater.cc:464] Registering TS with master...
08:21:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.706266 26969 heartbeater.cc:511] Master 127.25.233.61:36305 requested a full tablet report, sending...
08:21:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.711516 26653 ts_manager.cc:194] Registered new tserver with Master: aa030e46e2684533922e85bd94aa50ec (127.25.233.1:44721)
08:21:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.724653 26973 heartbeater.cc:347] Connected to a master server at 127.25.233.62:33477
08:21:32.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.725082 26973 heartbeater.cc:464] Registering TS with master...
08:21:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.726078 26973 heartbeater.cc:511] Master 127.25.233.62:33477 requested a full tablet report, sending...
08:21:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.728571 26577 ts_manager.cc:194] Registered new tserver with Master: aa030e46e2684533922e85bd94aa50ec (127.25.233.1:44721)
08:21:32.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:32.731339 26577 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.25.233.1:36011
08:21:33.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.126511 26972 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:33.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.127063 26972 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:33.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.127723 26972 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:33.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.160176 26972 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:33.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.161163 26972 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.233.2
08:21:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:33.197161 26972 tablet_server_runner.cc:78] Tablet server non-default flags:
08:21:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/data
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/wal
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.2:0
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/data/info.pb
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.2
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.233.2
08:21:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/logs
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:33.198761 26972 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:33.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:33.201118 26972 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:33.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.220986 26984 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.575472 26764 debug-util.cc:398] Leaking SignalData structure 0x7b08000c0ec0 after lost signal to thread 26695
08:21:33.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:33.735858 26973 heartbeater.cc:503] Master 127.25.233.62:33477 was elected leader, sending a full tablet report...
08:21:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.224090 26986 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:33.225304 26983 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:33.223029 26972 server_base.cc:1034] running on GCE node
08:21:34.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.504890 26972 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.508186 26972 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:34.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.509740 26972 hybrid_clock.cc:648] HybridClock initialized: now 1714638094509701 us; error 51 us; skew 500 ppm
08:21:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.510777 26972 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:34.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.519012 26972 webserver.cc:460] Webserver started at http://127.25.233.2:40603/ using document root <none> and password file <none>
08:21:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.520057 26972 fs_manager.cc:359] Metadata directory not provided
08:21:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.520273 26972 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.520707 26972 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.525943 26972 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/data/instance:
08:21:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e21e5d2bc0749a881d68bc8142a9ea7"
08:21:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:34 on dist-test-slave-rhmj"
08:21:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.527424 26972 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/wal/instance:
08:21:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e21e5d2bc0749a881d68bc8142a9ea7"
08:21:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:34 on dist-test-slave-rhmj"
08:21:34.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.537276 26972 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
08:21:34.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.543880 26997 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:34.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.545145 26972 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.545536 26972 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/wal
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e21e5d2bc0749a881d68bc8142a9ea7"
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:34 on dist-test-slave-rhmj"
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.545967 26972 fs_report.cc:383] FS layout report
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/wal
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/wal
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/data/data
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:34.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:34.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:34.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:34.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.604172 26972 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:34.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.606403 26972 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:34.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.606979 26972 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:34.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.610231 26972 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:21:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.615893 26972 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:21:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.616125 26972 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:21:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.616438 26972 ts_tablet_manager.cc:590] Registered 0 tablets
08:21:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.616628 26972 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:21:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.816478 26972 rpc_server.cc:307] RPC server started. Bound to: 127.25.233.2:44305
08:21:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.816676 27110 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.233.2:44305 every 8 connection(s)
08:21:34.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.819203 26972 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/data/info.pb
08:21:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.828619 26532 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 26972
08:21:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.829456 26532 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-1/wal/instance
08:21:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.851028 26532 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/wal
08:21:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/data
08:21:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/logs
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/data/info.pb
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.3:0
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.233.3
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.3
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:21:34.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.877404 27111 heartbeater.cc:347] Connected to a master server at 127.25.233.60:38453
08:21:34.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.877893 27111 heartbeater.cc:464] Registering TS with master...
08:21:34.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.879397 27111 heartbeater.cc:511] Master 127.25.233.60:38453 requested a full tablet report, sending...
08:21:34.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.883317 26733 ts_manager.cc:194] Registered new tserver with Master: 8e21e5d2bc0749a881d68bc8142a9ea7 (127.25.233.2:44305)
08:21:34.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.887571 27113 heartbeater.cc:347] Connected to a master server at 127.25.233.62:33477
08:21:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.888025 27113 heartbeater.cc:464] Registering TS with master...
08:21:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.888375 27112 heartbeater.cc:347] Connected to a master server at 127.25.233.61:36305
08:21:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.888819 27112 heartbeater.cc:464] Registering TS with master...
08:21:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.889029 27113 heartbeater.cc:511] Master 127.25.233.62:33477 requested a full tablet report, sending...
08:21:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.889881 27112 heartbeater.cc:511] Master 127.25.233.61:36305 requested a full tablet report, sending...
08:21:34.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.891624 26577 ts_manager.cc:194] Registered new tserver with Master: 8e21e5d2bc0749a881d68bc8142a9ea7 (127.25.233.2:44305)
08:21:34.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.893594 26577 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.25.233.2:41049
08:21:34.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:34.894227 26653 ts_manager.cc:194] Registered new tserver with Master: 8e21e5d2bc0749a881d68bc8142a9ea7 (127.25.233.2:44305)
08:21:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:35.198375 27121 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:35.198863 27121 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:35.199311 27121 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:35.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:35.230988 27121 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:35.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:35.231873 27121 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.233.3
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:35.267094 27121 tablet_server_runner.cc:78] Tablet server non-default flags:
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.233.20:38979
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/data
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/wal
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.233.3:0
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/data/info.pb
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.233.3
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.233.62:33477,127.25.233.61:36305,127.25.233.60:38453
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.233.3
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/logs
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:35.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:35.268359 27121 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:35.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:35.269958 27121 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:35.290361 27128 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:35.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:35.897991 27113 heartbeater.cc:503] Master 127.25.233.62:33477 was elected leader, sending a full tablet report...
08:21:36.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:35.291122 27127 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:36.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:35.291939 27130 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:36.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:35.292698 27121 server_base.cc:1034] running on GCE node
08:21:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.453346 27121 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:36.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.456429 27121 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:36.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.457906 27121 hybrid_clock.cc:648] HybridClock initialized: now 1714638096457853 us; error 69 us; skew 500 ppm
08:21:36.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.458724 27121 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:36.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.465332 27121 webserver.cc:460] Webserver started at http://127.25.233.3:44053/ using document root <none> and password file <none>
08:21:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.466238 27121 fs_manager.cc:359] Metadata directory not provided
08:21:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.466439 27121 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:36.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.466848 27121 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.471899 27121 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/data/instance:
08:21:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51af47be88442ee9195cafd43f77c65"
08:21:36.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:36 on dist-test-slave-rhmj"
08:21:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.473248 27121 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/wal/instance:
08:21:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51af47be88442ee9195cafd43f77c65"
08:21:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:36 on dist-test-slave-rhmj"
08:21:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.481544 27121 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.003s
08:21:36.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.487985 27139 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:36.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.489238 27121 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:21:36.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.489552 27121 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/wal
08:21:36.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f51af47be88442ee9195cafd43f77c65"
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:36 on dist-test-slave-rhmj"
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.489881 27121 fs_report.cc:383] FS layout report
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/wal
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/wal
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/data/data
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:36.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.544466 27121 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:36.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.546159 27121 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:36.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.546617 27121 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.549167 27121 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:21:36.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.553577 27121 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:21:36.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.553781 27121 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:21:36.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.553969 27121 ts_tablet_manager.cc:590] Registered 0 tablets
08:21:36.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.554113 27121 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:21:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.700172 27121 rpc_server.cc:307] RPC server started. Bound to: 127.25.233.3:44741
08:21:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.700284 27252 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.233.3:44741 every 8 connection(s)
08:21:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.702752 27121 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/data/info.pb
08:21:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.710901 26532 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 27121
08:21:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.711401 26532 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster4948298403073656402/ts-2/wal/instance
08:21:36.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.746305 27253 heartbeater.cc:347] Connected to a master server at 127.25.233.60:38453
08:21:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.746861 27253 heartbeater.cc:464] Registering TS with master...
08:21:36.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.748225 27253 heartbeater.cc:511] Master 127.25.233.60:38453 requested a full tablet report, sending...
08:21:36.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.751508 27254 heartbeater.cc:347] Connected to a master server at 127.25.233.61:36305
08:21:36.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.751487 26733 ts_manager.cc:194] Registered new tserver with Master: f51af47be88442ee9195cafd43f77c65 (127.25.233.3:44741)
08:21:36.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.751860 27254 heartbeater.cc:464] Registering TS with master...
08:21:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.752722 27254 heartbeater.cc:511] Master 127.25.233.61:36305 requested a full tablet report, sending...
08:21:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.754299 27259 heartbeater.cc:347] Connected to a master server at 127.25.233.62:33477
08:21:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.754654 27259 heartbeater.cc:464] Registering TS with master...
08:21:36.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.755525 27259 heartbeater.cc:511] Master 127.25.233.62:33477 requested a full tablet report, sending...
08:21:36.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.756631 26652 ts_manager.cc:194] Registered new tserver with Master: f51af47be88442ee9195cafd43f77c65 (127.25.233.3:44741)
08:21:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.757666 26576 ts_manager.cc:194] Registered new tserver with Master: f51af47be88442ee9195cafd43f77c65 (127.25.233.3:44741)
08:21:36.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.759032 26576 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.25.233.3:59977
08:21:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:36.778723 26532 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
08:21:36.790 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:21:36.792 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:21:36.803 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.233.62"
port: 33477
}
bound_http_address {
host: "127.25.233.62"
port: 41807
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.233.61"
port: 36305
}
bound_http_address {
host: "127.25.233.61"
port: 39993
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.233.60"
port: 38453
}
bound_http_address {
host: "127.25.233.60"
port: 33693
}
}
}
08:21:36.808 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:21:36.811 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.25.233.1"
port: 44721
}
bound_http_address {
host: "127.25.233.1"
port: 34673
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.25.233.2"
port: 44305
}
bound_http_address {
host: "127.25.233.2"
port: 40603
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.25.233.3"
port: 44741
}
bound_http_address {
host: "127.25.233.3"
port: 44053
}
}
}
08:21:36.811 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:21:37.011 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:21:37.149 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskcW3gy0/test-tmp
08:21:37.150 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:21:37.150 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskcW3gy0/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:21:37.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:21:37.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:37.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:21:37.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:37.510581 27265 mini_oidc.cc:152] Starting JWKS server
08:21:37.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:37.514178 27265 webserver.cc:460] Webserver started at http://127.0.0.1:43143/ using document root <none> and password file <none>
08:21:37.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:37.514595 27265 mini_oidc.cc:189] Starting OIDC Discovery server
08:21:37.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:37.516314 27265 webserver.cc:460] Webserver started at http://127.0.0.1:42029/ using document root <none> and password file <none>
08:21:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:21:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/principal' for realm 'KRBTEST.COM',
08:21:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:21:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:37 dist-test-slave-rhmj krb5kdc[27278](info): setting up network...
08:21:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:21:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:37 dist-test-slave-rhmj krb5kdc[27278](info): set up 2 sockets
08:21:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:37 dist-test-slave-rhmj krb5kdc[27278](info): commencing operation
08:21:37.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:21:37.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:37.761893 27259 heartbeater.cc:503] Master 127.25.233.62:33477 was elected leader, sending a full tablet report...
08:21:38.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:38.567478 27265 mini_kdc.cc:121] Time spent starting KDC: real 1.009s user 0.007s sys 0.013s
08:21:38.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:21:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:21:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:21:38.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:21:38.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:21:38.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:21:38.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:21:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:21:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:21:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:21:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/test-user.keytab.
08:21:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/test-user.keytab.
08:21:38.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:38 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638098, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:38.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:21:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T08:21:38Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:21:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T08:21:38Z Disabled control of system clock
08:21:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.26.160.126@KRBTEST.COM; defaulting to no policy
08:21:38.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:21:38.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.26.160.126@KRBTEST.COM" created.
08:21:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:21:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.160.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.126.keytab.
08:21:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.160.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.126.keytab.
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:38.784845 27265 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/logs
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/info.pb
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.126:38095
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.126
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.126.keytab
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.126
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:21:39.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.112545 27291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:39.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.113233 27291 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.113953 27291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.146349 27291 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:21:39.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.146626 27291 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.147032 27291 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.147347 27291 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.147670 27291 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.147972 27291 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:39.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.148211 27291 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43143/jwks.json
08:21:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:39.185286 27291 master_runner.cc:386] Master server non-default flags:
08:21:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.126.keytab
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.126
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.126:38095
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/info.pb
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.126
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/logs
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:39.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:39.186785 27291 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:39.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:39.188045 27291 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.204339 27297 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.204811 27298 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:39.207056 27291 server_base.cc:1034] running on GCE node
08:21:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:39.205672 27300 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.369642 27291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.372707 27291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:40.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.374234 27291 hybrid_clock.cc:648] HybridClock initialized: now 1714638100374206 us; error 47 us; skew 500 ppm
08:21:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.375028 27291 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:40.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:40 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638100, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:40.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.381269 27291 init.cc:377] Logged in from keytab as kudu/127.26.160.126@KRBTEST.COM (short username kudu)
08:21:40.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.386569 27291 webserver.cc:460] Webserver started at http://127.26.160.126:34825/ using document root <none> and password file <none>
08:21:40.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.387455 27291 fs_manager.cc:359] Metadata directory not provided
08:21:40.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.387686 27291 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.388108 27291 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.393551 27291 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/instance:
08:21:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f5b8c0ef7954bc88f028868dd331c05"
08:21:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:40 on dist-test-slave-rhmj"
08:21:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.394714 27291 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal/instance:
08:21:40.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f5b8c0ef7954bc88f028868dd331c05"
08:21:40.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:40 on dist-test-slave-rhmj"
08:21:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.402771 27291 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
08:21:40.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.408847 27309 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.409914 27291 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:21:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.410216 27291 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f5b8c0ef7954bc88f028868dd331c05"
08:21:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:40 on dist-test-slave-rhmj"
08:21:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.410537 27291 fs_report.cc:383] FS layout report
08:21:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/data
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:40.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.484437 27291 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:40.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.490265 27291 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:40.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.490728 27291 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.560878 27291 rpc_server.cc:307] RPC server started. Bound to: 127.26.160.126:38095
08:21:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.560973 27361 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.26.160.126:38095 every 8 connection(s)
08:21:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.563431 27291 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/info.pb
08:21:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.568800 27265 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 27291
08:21:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.568920 27362 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:21:40.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.569278 27265 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal/instance
08:21:40.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.581279 27362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } has no permanent_uuid. Determining permanent_uuid...
08:21:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.26.160.125@KRBTEST.COM; defaulting to no policy
08:21:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:21:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.26.160.125@KRBTEST.COM" created.
08:21:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:40 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638100, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.126@KRBTEST.COM for kudu/127.26.160.126@KRBTEST.COM
08:21:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:21:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.160.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.125.keytab.
08:21:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.160.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.125.keytab.
08:21:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.620374 27265 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/logs
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/info.pb
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.125:33603
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.125
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.125.keytab
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.125
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:21:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.620674 27362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } has no permanent_uuid. Determining permanent_uuid...
08:21:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.623319 27313 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:40.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.630328 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.125:33603: Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.679824 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } attempt: 1
08:21:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.684075 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.125:33603: Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:40.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.752516 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } attempt: 2
08:21:40.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.756248 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.125:33603: Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:40.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.847739 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } attempt: 3
08:21:40.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.851816 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.125:33603: Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.954617 27369 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.955165 27369 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.955803 27369 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.986989 27369 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:21:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.987226 27369 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.987491 27369 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.987743 27369 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.988035 27369 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.988248 27369 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.988401 27369 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43143/jwks.json
08:21:40.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:40.995311 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } attempt: 4
08:21:40.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:40.999150 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.125:33603: Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:41.026330 27369 master_runner.cc:386] Master server non-default flags:
08:21:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data
08:21:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.125.keytab
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.125
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.125:33603
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/info.pb
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.125
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/logs
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:41.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:41.027566 27369 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:41.028661 27369 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:41.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:41.046159 27378 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:41.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:41.298607 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } attempt: 5
08:21:41.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:41.302999 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.125:33603: Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:41.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:41.850616 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } attempt: 6
08:21:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:41.858263 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.125:33603: Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:42.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:41.046531 27375 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:42.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:41.048182 27369 server_base.cc:1034] running on GCE node
08:21:42.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:41.047494 27376 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:42.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.211297 27369 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:42.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.214000 27369 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:42.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.215409 27369 hybrid_clock.cc:648] HybridClock initialized: now 1714638102215356 us; error 58 us; skew 500 ppm
08:21:42.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.216248 27369 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:42 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638102, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.228176 27369 init.cc:377] Logged in from keytab as kudu/127.26.160.125@KRBTEST.COM (short username kudu)
08:21:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.233533 27369 webserver.cc:460] Webserver started at http://127.26.160.125:34693/ using document root <none> and password file <none>
08:21:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.234413 27369 fs_manager.cc:359] Metadata directory not provided
08:21:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.234611 27369 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.235108 27369 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.241657 27369 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/instance:
08:21:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b4727e92bb1481ea0f728e7fd32880d"
08:21:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:42 on dist-test-slave-rhmj"
08:21:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.242833 27369 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal/instance:
08:21:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b4727e92bb1481ea0f728e7fd32880d"
08:21:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:42 on dist-test-slave-rhmj"
08:21:42.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.250249 27369 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
08:21:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.255649 27388 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:42.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.256717 27369 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
08:21:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.257097 27369 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b4727e92bb1481ea0f728e7fd32880d"
08:21:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:42 on dist-test-slave-rhmj"
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.257560 27369 fs_report.cc:383] FS layout report
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/data
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.313792 27369 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.320010 27369 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.320477 27369 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:42.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.389140 27369 rpc_server.cc:307] RPC server started. Bound to: 127.26.160.125:33603
08:21:42.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.389286 27440 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.26.160.125:33603 every 8 connection(s)
08:21:42.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.392239 27369 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/info.pb
08:21:42.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.394583 27265 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 27369
08:21:42.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.395027 27265 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal/instance
08:21:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.400802 27441 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:21:42.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.26.160.124@KRBTEST.COM; defaulting to no policy
08:21:42.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.417582 27441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } has no permanent_uuid. Determining permanent_uuid...
08:21:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:21:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.26.160.124@KRBTEST.COM" created.
08:21:42.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:42 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638102, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.125@KRBTEST.COM for kudu/127.26.160.126@KRBTEST.COM
08:21:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:21:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.160.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.124.keytab.
08:21:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.160.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.124.keytab.
08:21:42.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.452560 27441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } has no permanent_uuid. Determining permanent_uuid...
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.453763 27265 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/logs
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/info.pb
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.124:45237
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.124
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.124.keytab
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.124
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:21:42.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:42 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638102, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.125@KRBTEST.COM for kudu/127.26.160.125@KRBTEST.COM
08:21:42.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.477185 27441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } has no permanent_uuid. Determining permanent_uuid...
08:21:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.479751 27391 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.484134 27441 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:42.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.533696 27441 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 1
08:21:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.538388 27441 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:42.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.606802 27441 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 2
08:21:42.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.611635 27441 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.703105 27441 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 3
08:21:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.707396 27441 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.773038 27448 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.773846 27448 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:42.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.774608 27448 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.809165 27448 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:21:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.809392 27448 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.809669 27448 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.809867 27448 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.810058 27448 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.810245 27448 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.810384 27448 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43143/jwks.json
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.846935 27448 master_runner.cc:386] Master server non-default flags:
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.124.keytab
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.124
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.124:45237
08:21:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/info.pb
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.124
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/logs
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.848136 27448 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:42.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.849185 27448 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:42.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.850879 27441 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 4
08:21:42.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.855257 27441 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:42.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.866647 27456 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:42.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.918926 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } attempt: 7
08:21:42.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:42 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638100, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.126@KRBTEST.COM for kudu/127.26.160.125@KRBTEST.COM
08:21:42.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.950634 27362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } has no permanent_uuid. Determining permanent_uuid...
08:21:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.958029 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:43.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:43.016440 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 1
08:21:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:43.021855 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:43.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:43.103271 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 2
08:21:43.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:43.108990 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:43.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:43.154629 27441 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 5
08:21:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:43.158442 27441 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:43.194419 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 3
08:21:43.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:43.199257 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:43.339686 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 4
08:21:43.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:43.344322 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:43.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:43.627728 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 5
08:21:43.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:43.632342 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:43.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:43.705845 27441 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 6
08:21:43.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:43.712549 27441 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.867271 27458 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:42.868301 27455 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:42.868957 27448 server_base.cc:1034] running on GCE node
08:21:44.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.036531 27448 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:44.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.039216 27448 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:44.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.040582 27448 hybrid_clock.cc:648] HybridClock initialized: now 1714638104040530 us; error 83 us; skew 500 ppm
08:21:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.041400 27448 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:44.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638104, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.047477 27448 init.cc:377] Logged in from keytab as kudu/127.26.160.124@KRBTEST.COM (short username kudu)
08:21:44.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.052845 27448 webserver.cc:460] Webserver started at http://127.26.160.124:46825/ using document root <none> and password file <none>
08:21:44.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.053716 27448 fs_manager.cc:359] Metadata directory not provided
08:21:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.053903 27448 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.054270 27448 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:21:44.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.059168 27448 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/instance:
08:21:44.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9f035df151640b7bb4e52b093ce5063"
08:21:44.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:44 on dist-test-slave-rhmj"
08:21:44.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.060286 27448 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal/instance:
08:21:44.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9f035df151640b7bb4e52b093ce5063"
08:21:44.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:44 on dist-test-slave-rhmj"
08:21:44.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.068608 27448 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
08:21:44.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.074168 27474 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:44.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.075301 27448 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
08:21:44.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.075649 27448 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9f035df151640b7bb4e52b093ce5063"
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:44 on dist-test-slave-rhmj"
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.075980 27448 fs_report.cc:383] FS layout report
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/data
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:44.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:44.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.145076 27448 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.151106 27448 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.151546 27448 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:44.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.184764 27362 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } attempt: 6
08:21:44.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.193136 27362 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.26.160.124:45237: Network error: Client connection negotiation failed: client connection to 127.26.160.124:45237: connect: Connection refused (error 111)
08:21:44.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.224557 27448 rpc_server.cc:307] RPC server started. Bound to: 127.26.160.124:45237
08:21:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.224617 27528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.26.160.124:45237 every 8 connection(s)
08:21:44.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.227228 27448 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/info.pb
08:21:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.231395 27265 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 27448
08:21:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.231948 27265 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal/instance
08:21:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.233373 27529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:21:44.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.246762 27529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } has no permanent_uuid. Determining permanent_uuid...
08:21:44.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638098, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.160.124@KRBTEST.COM
08:21:44.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638104, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.124@KRBTEST.COM for kudu/127.26.160.126@KRBTEST.COM
08:21:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.280164 27529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } has no permanent_uuid. Determining permanent_uuid...
08:21:44.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638104, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.124@KRBTEST.COM for kudu/127.26.160.125@KRBTEST.COM
08:21:44.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638098, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.160.125@KRBTEST.COM
08:21:44.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.304024 27529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } has no permanent_uuid. Determining permanent_uuid...
08:21:44.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638104, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.124@KRBTEST.COM for kudu/127.26.160.124@KRBTEST.COM
08:21:44.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638098, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.160.126@KRBTEST.COM
08:21:44.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.334215 27529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Bootstrap starting.
08:21:44.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.338083 27265 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
08:21:44.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.339851 27529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Neither blocks nor log segments found. Creating new log.
08:21:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.341609 27529 log.cc:824] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Log is configured to *not* fsync() on all Append() calls
08:21:44.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:44.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.346522 27529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: No bootstrap required, opened a new log
08:21:44.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:21:44.356 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:21:44.357 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:21:44.360 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.26.160.126"
port: 38095
}
bound_http_address {
host: "127.26.160.126"
port: 34825
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.26.160.125"
port: 33603
}
bound_http_address {
host: "127.26.160.125"
port: 34693
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.26.160.124"
port: 45237
}
bound_http_address {
host: "127.26.160.124"
port: 46825
}
}
}
08:21:44.360 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:21:44.361 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:44.361 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:21:44.362 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:21:44.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.369418 27529 raft_consensus.cc:348] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:21:44.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.370071 27529 raft_consensus.cc:374] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:21:44.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.370369 27529 raft_consensus.cc:729] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9f035df151640b7bb4e52b093ce5063, State: Initialized, Role: FOLLOWER
08:21:44.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.371255 27529 consensus_queue.cc:260] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:21:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.374265 27539 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } } }
08:21:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.375169 27539 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [sys.catalog]: This master's current role is: FOLLOWER
08:21:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.376134 27529 sys_catalog.cc:564] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [sys.catalog]: configured and running, proceeding with master startup.
08:21:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638104, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:21:44.400 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:44.402 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc
08:21:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.406906 27480 tablet.cc:2369] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:21:44.415 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:21:44.416 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc
08:21:44.421 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:21:44.434 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:21:44.439 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:21:44.439 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:21:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.470264 27551 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:44.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.470635 27551 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:44.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:44 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638104, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:44.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:21:44.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.451908 27448 master_runner.cc:186] Error getting master registration for 127.26.160.126:38095: OK, instance_id { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" instance_seqno: 1714638100541853 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:21:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.484038 27448 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:21:44.489 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:44.489 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc
08:21:44.490 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:21:44.491 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.160.126:38095
08:21:44.492 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:21:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.493495 27265 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 27291
08:21:44.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:44.523 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.160.125:33603
08:21:44.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:21:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.524085 27265 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 27369
08:21:44.551 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:44.551 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.160.124:45237
08:21:44.551 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:21:44.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.553161 27265 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 27448
08:21:44.574 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:44.574 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.160.126:38095
08:21:44.576 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:21:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.577823 27265 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/logs
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/info.pb
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.126:38095
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.126
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34825
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.126.keytab
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.126
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:21:44.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.907634 27555 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:44.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.908339 27555 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.909080 27555 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:44.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.943095 27555 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:21:44.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.943393 27555 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.943820 27555 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.944131 27555 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.944422 27555 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.944712 27555 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:44.944954 27555 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43143/jwks.json
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.981643 27555 master_runner.cc:386] Master server non-default flags:
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.126.keytab
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.126
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:44.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.126:38095
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/info.pb
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.126
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34825
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/logs
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.983157 27555 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:44.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:44.984714 27555 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:45.002233 27561 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:45.003072 27562 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:45.004655 27555 server_base.cc:1034] running on GCE node
08:21:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:45.004171 27564 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:46.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.155417 27555 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:46.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.158205 27555 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:46.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.159641 27555 hybrid_clock.cc:648] HybridClock initialized: now 1714638106159547 us; error 62 us; skew 500 ppm
08:21:46.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.160537 27555 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:46.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:46 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638106, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:46.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.166846 27555 init.cc:377] Logged in from keytab as kudu/127.26.160.126@KRBTEST.COM (short username kudu)
08:21:46.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.172276 27555 webserver.cc:460] Webserver started at http://127.26.160.126:34825/ using document root <none> and password file <none>
08:21:46.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.173523 27555 fs_manager.cc:359] Metadata directory not provided
08:21:46.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.173745 27555 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.182058 27555 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.002s
08:21:46.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.186697 27574 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:46.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.187917 27555 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:21:46.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.188256 27555 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:46.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f5b8c0ef7954bc88f028868dd331c05"
08:21:46.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:40 on dist-test-slave-rhmj"
08:21:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.190735 27555 fs_report.cc:383] FS layout report
08:21:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal
08:21:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/data
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.258903 27555 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:46.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.265324 27555 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.265900 27555 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:46.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.337132 27555 rpc_server.cc:307] RPC server started. Bound to: 127.26.160.126:38095
08:21:46.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.337231 27626 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.26.160.126:38095 every 8 connection(s)
08:21:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.340009 27555 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/data/info.pb
08:21:46.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.345818 27265 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 27555
08:21:46.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:46.346 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.160.125:33603
08:21:46.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:21:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.348414 27265 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/logs
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/info.pb
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.125:33603
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.125
08:21:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34693
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.125.keytab
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.125
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:21:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.351554 27627 sys_catalog.cc:263] Verifying existing consensus state
08:21:46.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 08:21:46.352337 27627 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:21:46.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.352730 27555 master.cc:537] Master@127.26.160.126:38095 shutting down...
08:21:46.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.420742 27555 master.cc:552] Master@127.26.160.126:38095 shutdown complete.
08:21:46.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:21:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.672380 27629 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.673177 27629 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.674042 27629 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.705224 27629 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:21:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.705457 27629 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.705726 27629 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:46.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.705929 27629 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:46.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.706121 27629 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:46.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.706308 27629 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:46.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.706449 27629 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43143/jwks.json
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.742929 27629 master_runner.cc:386] Master server non-default flags:
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.125.keytab
08:21:46.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.125
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.125:33603
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/info.pb
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.125
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34693
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/logs
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.744171 27629 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:46.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.745366 27629 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:46.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.761689 27635 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.762552 27636 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:46.763268 27638 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:46.764137 27629 server_base.cc:1034] running on GCE node
08:21:47.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.918994 27629 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:47.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.921794 27629 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:47.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.923197 27629 hybrid_clock.cc:648] HybridClock initialized: now 1714638107923152 us; error 54 us; skew 500 ppm
08:21:47.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.924144 27629 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:47.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:47 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638107, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:47.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.936710 27629 init.cc:377] Logged in from keytab as kudu/127.26.160.125@KRBTEST.COM (short username kudu)
08:21:47.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.942708 27629 webserver.cc:460] Webserver started at http://127.26.160.125:34693/ using document root <none> and password file <none>
08:21:47.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.944479 27629 fs_manager.cc:359] Metadata directory not provided
08:21:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.944783 27629 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:47.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.953725 27629 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.003s
08:21:47.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.958107 27648 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:47.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.959251 27629 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:21:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.959573 27629 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b4727e92bb1481ea0f728e7fd32880d"
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:42 on dist-test-slave-rhmj"
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:47.961966 27629 fs_report.cc:383] FS layout report
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/data
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:48.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.029028 27629 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:48.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.035480 27629 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:48.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.036003 27629 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.110432 27629 rpc_server.cc:307] RPC server started. Bound to: 127.26.160.125:33603
08:21:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.110536 27700 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.26.160.125:33603 every 8 connection(s)
08:21:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.114246 27629 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/data/info.pb
08:21:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.119390 27265 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 27629
08:21:48.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:48.121 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.160.124:45237
08:21:48.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:21:48.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.123224 27265 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:48.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/logs
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/info.pb
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.124:45237
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.124
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46825
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.124.keytab
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.124
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:21:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.128793 27701 sys_catalog.cc:263] Verifying existing consensus state
08:21:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 08:21:48.130021 27701 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:21:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.130556 27629 master.cc:537] Master@127.26.160.125:33603 shutting down...
08:21:48.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.189586 27629 master.cc:552] Master@127.26.160.125:33603 shutdown complete.
08:21:48.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:21:48.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.440990 27703 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:21:48.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.441558 27703 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:21:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.442124 27703 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:21:48.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.473044 27703 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:21:48.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.473325 27703 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:21:48.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.473594 27703 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:21:48.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.473809 27703 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:21:48.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.474217 27703 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:21:48.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.474448 27703 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:21:48.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.474594 27703 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43143/jwks.json
08:21:48.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.512049 27703 master_runner.cc:386] Master server non-default flags:
08:21:48.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.160.84:34853
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/krb5kdc/kudu_127.26.160.124.keytab
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.160.124
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.160.124:45237
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43143/jwks.json
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/info.pb
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.160.124
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46825
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/logs
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:21:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:21:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:21:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:21:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 07:43:23 UTC on 1a15035a6830
08:21:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 88
08:21:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:21:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.513200 27703 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:21:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.514204 27703 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:21:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.529827 27710 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.531816 27712 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:48.532279 27703 server_base.cc:1034] running on GCE node
08:21:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:48.530691 27709 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:21:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.676180 27703 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:21:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.678694 27703 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:21:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.680040 27703 hybrid_clock.cc:648] HybridClock initialized: now 1714638109679993 us; error 53 us; skew 500 ppm
08:21:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.680900 27703 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:21:49.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:49 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638109, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.687024 27703 init.cc:377] Logged in from keytab as kudu/127.26.160.124@KRBTEST.COM (short username kudu)
08:21:49.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.692549 27703 webserver.cc:460] Webserver started at http://127.26.160.124:46825/ using document root <none> and password file <none>
08:21:49.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.693475 27703 fs_manager.cc:359] Metadata directory not provided
08:21:49.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.693697 27703 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:21:49.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.701802 27703 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
08:21:49.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.706472 27721 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:21:49.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.707878 27703 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:21:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.708245 27703 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data,/tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e9f035df151640b7bb4e52b093ce5063"
08:21:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 08:21:44 on dist-test-slave-rhmj"
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.710578 27703 fs_report.cc:383] FS layout report
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/wal
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/data
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:21:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:21:49.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.760720 27703 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:21:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.766546 27703 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:21:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.767069 27703 kserver.cc:163] Server-wide thread pool size limit: 3276
08:21:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.853752 27703 rpc_server.cc:307] RPC server started. Bound to: 127.26.160.124:45237
08:21:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.853922 27772 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.26.160.124:45237 every 8 connection(s)
08:21:49.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.856489 27703 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskcW3gy0/test-tmp/mini-kudu-cluster8770642893216373675/master-2/data/info.pb
08:21:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.860734 27265 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu as pid 27703
08:21:49.861 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:21:49.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.869915 27773 sys_catalog.cc:263] Verifying existing consensus state
08:21:49.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.878232 27773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Bootstrap starting.
08:21:49.915 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:49.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.916316 27773 log.cc:824] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Log is configured to *not* fsync() on all Append() calls
08:21:49.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.921093 27773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:21:49.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.921849 27773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Bootstrap complete.
08:21:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.955883 27773 raft_consensus.cc:348] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:21:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.958875 27773 raft_consensus.cc:374] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:21:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.959195 27773 raft_consensus.cc:729] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e9f035df151640b7bb4e52b093ce5063, State: Initialized, Role: FOLLOWER
08:21:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.960033 27773 consensus_queue.cc:260] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:21:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.962862 27777 sys_catalog.cc:455] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } } }
08:21:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.963773 27777 sys_catalog.cc:458] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [sys.catalog]: This master's current role is: FOLLOWER
08:21:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:49.964632 27773 sys_catalog.cc:564] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [sys.catalog]: configured and running, proceeding with master startup.
08:21:49.984 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:49.986 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.160.124:45237(127.26.160.124:45237)] connecting to peer
08:21:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:50.004895 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:50.005263 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:50.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:50.022292 27727 tablet.cc:2369] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:21:50.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:50.002368 27703 master_runner.cc:186] Error getting master registration for 127.26.160.126:38095: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111),
08:21:50.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:50.035290 27703 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:21:50.041 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.042 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.044 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.047 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.160.124:45237(127.26.160.124:45237)] Successfully connected to peer
08:21:50.139 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:21:50.149 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.26.160.124
08:21:50.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:50.262696 27777 raft_consensus.cc:482] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:21:50.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:50.263268 27777 raft_consensus.cc:504] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:21:50.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:50.268662 27722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111)
08:21:50.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:50.271884 27777 leader_election.cc:288] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095), 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603)
08:21:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:50.278695 27722 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095): Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111)
08:21:50.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:50.283300 27725 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603): Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:50.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:50.283880 27725 leader_election.cc:302] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e9f035df151640b7bb4e52b093ce5063; no voters: 5b4727e92bb1481ea0f728e7fd32880d, 6f5b8c0ef7954bc88f028868dd331c05
08:21:50.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:50.284811 27777 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:21:50.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:50 dist-test-slave-rhmj krb5kdc[27278](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638104, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.160.124@KRBTEST.COM
08:21:50.384 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd980e8ec, L:/127.0.0.1:47982 - R:/127.26.160.124:45237] using SASL/GSSAPI
08:21:50.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.408 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.410 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.413 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.414 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.414 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.417 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.417 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.417 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.419 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.427 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.428 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.431 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.435 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.448 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.450 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.450 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.455 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.488 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.499 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.527 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.532 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.534 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.534 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.535 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.537 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.547 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.550 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.551 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.553 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.553 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.554 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.556 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.647 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.648 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.655 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.656 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.656 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.657 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.659 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.687 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.690 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.695 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:50.787 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:50.788 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:50.790 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.790 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.791 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:50.792 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:50.798 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:51.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:51.008433 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:51.008754 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:51.207 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:51.208 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:51.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:51.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:51.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:51.211 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:51.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:51.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:51.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:51.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:51.328 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:51.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:51.330 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:51.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:51.334 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:51.335 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:51.335 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:51.335 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:52.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:52.011731 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:52.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:52.012017 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:52.525921 27799 raft_consensus.cc:482] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:21:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:52.526335 27799 raft_consensus.cc:504] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:21:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:52.527899 27799 leader_election.cc:288] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095), 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603)
08:21:52.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:52.534304 27725 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603): Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:52.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:52.534699 27722 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095): Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111)
08:21:52.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:52.535040 27722 leader_election.cc:302] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e9f035df151640b7bb4e52b093ce5063; no voters: 5b4727e92bb1481ea0f728e7fd32880d, 6f5b8c0ef7954bc88f028868dd331c05
08:21:52.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:52.535560 27799 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:21:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:53.014834 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:53.015118 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:54.018105 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:54.018453 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:55.021440 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:55.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:55.021713 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:55.038064 27703 master_runner.cc:186] Error getting master registration for 127.26.160.126:38095: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111),
08:21:55.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:55.038466 27703 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:21:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:55.107899 27807 raft_consensus.cc:482] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:21:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:55.108274 27807 raft_consensus.cc:504] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:21:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:55.109774 27807 leader_election.cc:288] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095), 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603)
08:21:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:55.114493 27722 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095): Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111)
08:21:55.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:55.114903 27725 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603): Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:55.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:55.115195 27725 leader_election.cc:302] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e9f035df151640b7bb4e52b093ce5063; no voters: 5b4727e92bb1481ea0f728e7fd32880d, 6f5b8c0ef7954bc88f028868dd331c05
08:21:55.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:55.115725 27807 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:21:55.307 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:55.308 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:55.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:55.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:55.310 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:55.311 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:55.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:55.311 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:55.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:55.987 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:55.988 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:55.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:55.990 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:55.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:55.991 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:55.991 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:55.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:55.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:56.024647 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:56.024971 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:56.507 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:56.508 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:56.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:56.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:56.510 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:56.511 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:56.511 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:56.511 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:56.513 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:56.547 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:56.548 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:56.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:56.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:56.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:56.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:56.551 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:56.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:56.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:57.027863 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:57.028127 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:58.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:58.031152 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:58.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:58.031430 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:59.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:59.034950 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:21:59.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:59.035280 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:21:59.127 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:21:59.129 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:21:59.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:59.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:59.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:21:59.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:59.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:59.131 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:21:59.135 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:21:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:59.226002 27816 raft_consensus.cc:482] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:21:59.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:59.226456 27816 raft_consensus.cc:504] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:21:59.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:59.228114 27816 leader_election.cc:288] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095), 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603)
08:21:59.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:59.231015 27722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111) [suppressed 5 similar messages]
08:21:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:59.233909 27725 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603): Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:21:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:21:59.234382 27722 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095): Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111)
08:21:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:59.234704 27722 leader_election.cc:302] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e9f035df151640b7bb4e52b093ce5063; no voters: 5b4727e92bb1481ea0f728e7fd32880d, 6f5b8c0ef7954bc88f028868dd331c05
08:21:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:59.235297 27816 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:21:59.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:21:59 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638119, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:21:59.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:21:59.690937 27717 init.cc:295] Successfully reacquired a new kerberos TGT
08:22:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:00.038268 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:00.038545 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:00.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:00.042089 27703 master_runner.cc:186] Error getting master registration for 127.26.160.126:38095: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111),
08:22:00.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:00.042536 27703 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:22:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:01.041724 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:01.042006 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:01.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:01.489 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:01.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:01.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:01.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:01.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:01.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:01.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:01.494 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:02.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:02.045333 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:02.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:02.045742 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:03.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:03.048736 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:03.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:03.049095 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:03.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:03.444777 27823 raft_consensus.cc:482] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:22:03.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:03.445147 27823 raft_consensus.cc:504] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:22:03.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:03.446791 27823 leader_election.cc:288] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095), 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603)
08:22:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:03.451552 27722 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095): Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111)
08:22:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:03.452095 27725 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603): Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:22:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:03.452432 27725 leader_election.cc:302] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e9f035df151640b7bb4e52b093ce5063; no voters: 5b4727e92bb1481ea0f728e7fd32880d, 6f5b8c0ef7954bc88f028868dd331c05
08:22:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:03.452986 27823 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:22:03.767 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:03.768 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:03.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:03.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:03.770 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:03.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:03.771 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:03.771 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:03.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:04.052184 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:04.052485 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:04.467 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:04.468 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:04.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:04.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:04.470 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:04.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:04.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:04.471 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:04.473 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:05.045593 27703 master_runner.cc:186] Error getting master registration for 127.26.160.126:38095: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111),
08:22:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:05.045931 27703 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:22:05.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:05.055581 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:05.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:05.055909 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:06.058990 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:06.059250 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:06.427 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:06.428 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:06.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:06.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:06.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:06.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:06.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:06.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:06.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:07.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:07.062176 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:07.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:07.062445 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:07.600869 27834 raft_consensus.cc:482] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:22:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:07.601228 27834 raft_consensus.cc:504] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6f5b8c0ef7954bc88f028868dd331c05" member_type: VOTER last_known_addr { host: "127.26.160.126" port: 38095 } } peers { permanent_uuid: "5b4727e92bb1481ea0f728e7fd32880d" member_type: VOTER last_known_addr { host: "127.26.160.125" port: 33603 } } peers { permanent_uuid: "e9f035df151640b7bb4e52b093ce5063" member_type: VOTER last_known_addr { host: "127.26.160.124" port: 45237 } }
08:22:07.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:07.602962 27834 leader_election.cc:288] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095), 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603)
08:22:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:07.604911 27722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:22:07.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:07.607753 27725 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5b4727e92bb1481ea0f728e7fd32880d (127.26.160.125:33603): Network error: Client connection negotiation failed: client connection to 127.26.160.125:33603: connect: Connection refused (error 111)
08:22:07.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:07.608187 27722 leader_election.cc:334] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f5b8c0ef7954bc88f028868dd331c05 (127.26.160.126:38095): Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111)
08:22:07.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:07.608561 27722 leader_election.cc:302] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e9f035df151640b7bb4e52b093ce5063; no voters: 5b4727e92bb1481ea0f728e7fd32880d, 6f5b8c0ef7954bc88f028868dd331c05
08:22:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:07.609249 27834 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:22:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:08.065640 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:08.065922 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:09.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:09.069695 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:09.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:09.070027 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 08:22:09 dist-test-slave-rhmj krb5kdc[27278](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714638129, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.160.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:22:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:09.693382 27717 init.cc:295] Successfully reacquired a new kerberos TGT
08:22:10.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:10.049449 27703 master_runner.cc:186] Error getting master registration for 127.26.160.126:38095: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111),
08:22:10.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:10.049878 27703 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:22:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:10.073032 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:10.073299 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:10.227 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:10.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:10.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:10.228 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:10.229 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:10.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:10.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:10.231 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:10.234 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:11.076181 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:11.076442 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:12.079115 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:12.079387 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:12.507 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:12.508 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:12.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:12.508 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:12.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:12.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:12.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:12.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:12.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:13.007 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:13.007 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:13.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:13.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:13.008 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:13.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:13.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:13.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:13.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:13.082087 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:13.082394 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:14.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:14.085331 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:14.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:14.085580 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:15.052531 27703 master_runner.cc:186] Error getting master registration for 127.26.160.126:38095: Network error: Client connection negotiation failed: client connection to 127.26.160.126:38095: connect: Connection refused (error 111),
08:22:15.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:15.052868 27703 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:22:15.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:15.088459 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:15.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:15.088729 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:15.606130 26684 debug-util.cc:398] Leaking SignalData structure 0x7b08000a78e0 after lost signal to thread 26619
08:22:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:15.606729 26684 debug-util.cc:398] Leaking SignalData structure 0x7b0800095540 after lost signal to thread 26687
08:22:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:15.681721 26764 debug-util.cc:398] Leaking SignalData structure 0x7b080009fa40 after lost signal to thread 26695
08:22:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:15.682511 26764 debug-util.cc:398] Leaking SignalData structure 0x7b080007f200 after lost signal to thread 26767
08:22:16.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:16.091524 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:16.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:16.091835 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:16.147 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:16.148 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:16.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:16.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:16.148 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:16.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:16.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:16.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:16.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:17.094753 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:17.095034 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:17.667 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:17.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:17.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:17.668 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:17.668 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:17.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:17.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:17.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:17.672 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:18.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:18.098027 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:18.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:18.098325 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:19.102121 27792 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e9f035df151640b7bb4e52b093ce5063: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:22:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 08:22:19.102465 27792 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:22:19.287 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.126:38095(127.26.160.126:38095)] connecting to peer
08:22:19.288 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.26.160.125:33603(127.26.160.125:33603)] connecting to peer
08:22:19.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:19.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.160.126:38095(127.26.160.126:38095)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:19.288 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.126:38095: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095
08:22:19.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:19.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.125:33603(127.26.160.125:33603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:19.291 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.160.125:33603: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
08:22:19.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237; will retry
08:22:19.298 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29427), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [171ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [174ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [527ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [531ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [543ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [546ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [547ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [560ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [562ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [563ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [580ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [581ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [583ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [619ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [626ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [628ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [655ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [661ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [663ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [665ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [679ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [682ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [684ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [781ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [785ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [787ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [818ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [822ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [823ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [919ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [920ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [927ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1338ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1342ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1456ms] refreshing cache from master, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1464ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1464ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [5435ms] refreshing cache from master, [5435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [5436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [5438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [5438ms] too many traces: truncated at 100 traces, deferred=Deferred@745697118(state=PENDING, result=null, callback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703), errback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703)))
08:22:19.303 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.26.160.126:38095(127.26.160.126:38095)] disconnecting while in state TERMINATED
08:22:19.305 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.26.160.124:45237(127.26.160.124:45237)] disconnecting while in state READY
08:22:19.305 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.26.160.125:33603(127.26.160.125:33603)] disconnecting while in state TERMINATED
08:22:19.310 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:22:19.312 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.160.124:45237(127.26.160.124:45237)] handling channelInactive
08:22:19.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.160.124:45237(127.26.160.124:45237)] cleaning up while in state READY due to: connection closed
08:22:19.314 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:22:19.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:19.327840 27265 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 27703
08:22:20.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T08:22:20Z chronyd exiting
08:22:21.488 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:22:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:21.489763 26532 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 26816
08:22:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:21.516588 26532 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 26972
08:22:21.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:21.542595 26532 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 27121
08:22:21.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:21.569419 26532 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 26544
08:22:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:21.602682 26532 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 26615
08:22:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 08:22:21.634513 26532 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskcW3gy0/build/tsan/bin/kudu with pid 26690
08:22:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T08:22:22Z chronyd exiting
08:22:23.743 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29427), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [171ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [174ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [527ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [531ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [543ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [546ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [547ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [560ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [562ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [563ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [580ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [581ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [583ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [619ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [626ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [628ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [655ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [661ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [663ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [665ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [679ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [682ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [684ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [781ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [785ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [787ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [818ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [822ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [823ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [919ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [920ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [927ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1338ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1342ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1456ms] refreshing cache from master, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1464ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1464ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [5435ms] refreshing cache from master, [5435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [5436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [5438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [5438ms] too many traces: truncated at 100 traces, deferred=Deferred@745697118(state=PENDING, result=null, callback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703), errback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.26.160.125:33603
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29427), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [171ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [174ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [527ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [531ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [543ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [546ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [547ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [560ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [562ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [563ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [580ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [581ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [583ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [619ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [626ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [628ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [655ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [661ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [663ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [665ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [679ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [682ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [684ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [781ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [785ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [787ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [818ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [822ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [823ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [919ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [920ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [927ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1338ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1342ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1456ms] refreshing cache from master, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1464ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1464ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [5435ms] refreshing cache from master, [5435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [5436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [5438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [5438ms] too many traces: truncated at 100 traces, deferred=Deferred@745697118(state=PENDING, result=null, callback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703), errback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.26.160.125:33603
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
08:22:24.261 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
08:22:24.261 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29427), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [171ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [174ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [527ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [531ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [543ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [546ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [547ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [560ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [562ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [563ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [580ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [581ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [583ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [619ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [626ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [628ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [655ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [661ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [663ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [665ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [679ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [682ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [684ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [781ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [785ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [787ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [818ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [822ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [823ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [919ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [920ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [927ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1338ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1342ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1456ms] refreshing cache from master, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1464ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1464ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [5435ms] refreshing cache from master, [5435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [5436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [5438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [5438ms] too many traces: truncated at 100 traces, deferred=Deferred@745697118(state=PENDING, result=null, callback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703), errback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.26.160.125:33603
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29427), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [171ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [174ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [527ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [531ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [543ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [546ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [547ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [548ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [560ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [562ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [563ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [580ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [581ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [583ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [584ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [619ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [626ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [628ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [655ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [661ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [663ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [665ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [679ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [682ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [684ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [781ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [785ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [787ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [788ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [818ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [822ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [823ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [919ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [920ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [927ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [928ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1338ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1342ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1456ms] refreshing cache from master, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [1459ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.126:38095: Network error: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095, [1459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [1463ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.124:45237: OK, [1464ms] Sub RPC ConnectToMaster: received response from server master-127.26.160.125:33603: Network error: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [1464ms] delaying RPC due to: Service unavailable: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603, [5435ms] refreshing cache from master, [5435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.126:38095, [5436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.125:33603, [5438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.160.124:45237, [5438ms] too many traces: truncated at 100 traces, deferred=Deferred@745697118(state=PENDING, result=null, callback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703), errback=(continuation of Deferred@1905468617 after retry RPC after error@1326434499) -> (continuation of Deferred@606409767 after retry RPC after error@454203234) -> (continuation of Deferred@1934581157 after retry RPC after error@1119341279) -> (continuation of Deferred@1321171586 after retry RPC after error@1092181424) -> (continuation of Deferred@1219658307 after retry RPC after error@875518136) -> (continuation of Deferred@32595925 after retry RPC after error@177104518) -> (continuation of Deferred@1458724110 after retry RPC after error@408303088) -> (continuation of Deferred@324198104 after retry RPC after error@1787895906) -> (continuation of Deferred@785320467 after retry RPC after error@119493519) -> (continuation of Deferred@46070186 after retry RPC after error@1632586671) -> (continuation of Deferred@923163143 after retry RPC after error@1257829443) -> (continuation of Deferred@1073620767 after retry RPC after error@611778940) -> (continuation of Deferred@564437399 after retry RPC after error@745734924) -> (continuation of Deferred@1652236098 after retry RPC after error@2115739280) -> (continuation of Deferred@2050437341 after retry RPC after error@1323841524) -> (continuation of Deferred@1951637604 after retry RPC after error@1453215584) -> (continuation of Deferred@367803574 after retry RPC after error@439680616) -> (continuation of Deferred@1930247626 after retry RPC after error@1473022340) -> (continuation of Deferred@598808135 after retry RPC after error@508263399) -> (continuation of Deferred@70444681 after retry RPC after error@922865442) -> (continuation of Deferred@472511659 after retry RPC after error@1501048970) -> (continuation of Deferred@1257083778 after retry RPC after error@1932274978) -> (continuation of Deferred@544408384 after retry RPC after error@941067172) -> (continuation of Deferred@515931815 after retry RPC after error@211228000) -> (continuation of Deferred@885419697 after retry RPC after error@805869862) -> (continuation of Deferred@765521094 after retry RPC after error@430493703)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.26.160.126:38095,127.26.160.125:33603,127.26.160.124:45237) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.126:38095(127.26.160.126:38095): Connection refused: /127.26.160.126:38095,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.160.125:33603(127.26.160.125:33603): Connection refused: /127.26.160.125:33603
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.26.160.125:33603
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)