20:16:16.953 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:16:16.965 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskASVjAM/test-tmp
20:16:16.967 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:16:16.970 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskASVjAM/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:16:17.217 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:16:17.366 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:16:17.368 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:16:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.372375 6218 mini_oidc.cc:152] Starting JWKS server
20:16:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.375995 6218 webserver.cc:460] Webserver started at http://127.0.0.1:42903/ using document root <none> and password file <none>
20:16:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.376442 6218 mini_oidc.cc:189] Starting OIDC Discovery server
20:16:17.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.378583 6218 webserver.cc:460] Webserver started at http://127.0.0.1:45997/ using document root <none> and password file <none>
20:16:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T20:16:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:16:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T20:16:17Z Disabled control of system clock
20:16:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.444306 6218 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/wal
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/data
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/logs
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/data/info.pb
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.190:33169
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.190
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:17.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42903/jwks.json with env {}
20:16:17.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.746625 6230 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:17.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.747231 6230 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:17.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.747874 6230 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.778776 6230 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.779109 6230 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.779318 6230 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.779520 6230 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:17.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.779795 6230 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:17.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.780033 6230 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42903/jwks.json
20:16:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.815505 6230 master_runner.cc:386] Master server non-default flags:
20:16:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/data
20:16:17.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/wal
20:16:17.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:17.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:17.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:17.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:17.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:17.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.190:33169
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42903/jwks.json
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/data/info.pb
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.190
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/logs
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:17.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.816843 6230 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.819074 6230 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:17.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.837519 6239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.838078 6237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:17.839226 6230 server_base.cc:1034] running on GCE node
20:16:18.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:17.838551 6236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.962401 6230 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:18.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.965154 6230 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:18.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.966527 6230 hybrid_clock.cc:648] HybridClock initialized: now 1714508178966495 us; error 46 us; skew 500 ppm
20:16:18.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.967474 6230 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:18.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.973691 6230 webserver.cc:460] Webserver started at http://127.6.18.190:36847/ using document root <none> and password file <none>
20:16:18.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.974727 6230 fs_manager.cc:359] Metadata directory not provided
20:16:18.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.974948 6230 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:18.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.975476 6230 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.980548 6230 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/data/instance:
20:16:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a"
20:16:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:18 on dist-test-slave-7k6f"
20:16:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.981889 6230 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/wal/instance:
20:16:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a"
20:16:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:18 on dist-test-slave-7k6f"
20:16:18.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.989303 6230 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
20:16:18.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.994974 6246 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:18.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.996062 6230 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:16:18.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.996433 6230 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/wal
20:16:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a"
20:16:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:18 on dist-test-slave-7k6f"
20:16:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:18.996841 6230 fs_report.cc:383] FS layout report
20:16:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/wal
20:16:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/wal
20:16:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/data/data
20:16:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:19.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.054715 6230 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.056301 6230 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:19.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.056807 6230 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.125474 6230 rpc_server.cc:307] RPC server started. Bound to: 127.6.18.190:33169
20:16:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.125576 6297 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.18.190:33169 every 8 connection(s)
20:16:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.128358 6230 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/data/info.pb
20:16:19.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.130132 6218 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 6230
20:16:19.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.130782 6218 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-0/wal/instance
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.134889 6298 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.135128 6218 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/wal
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/data
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/logs
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/data/info.pb
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.189:38103
20:16:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.189
20:16:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42903/jwks.json with env {}
20:16:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.148015 6298 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } has no permanent_uuid. Determining permanent_uuid...
20:16:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.172971 6298 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } has no permanent_uuid. Determining permanent_uuid...
20:16:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.175595 6250 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.18.189:38103: connect: Connection refused (error 111)
20:16:19.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.179627 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.189:38103: Network error: Client connection negotiation failed: client connection to 127.6.18.189:38103: connect: Connection refused (error 111)
20:16:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.229117 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } attempt: 1
20:16:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.233372 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.189:38103: Network error: Client connection negotiation failed: client connection to 127.6.18.189:38103: connect: Connection refused (error 111)
20:16:19.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.301800 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } attempt: 2
20:16:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.305864 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.189:38103: Network error: Client connection negotiation failed: client connection to 127.6.18.189:38103: connect: Connection refused (error 111)
20:16:19.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.397276 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } attempt: 3
20:16:19.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.401446 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.189:38103: Network error: Client connection negotiation failed: client connection to 127.6.18.189:38103: connect: Connection refused (error 111)
20:16:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.447300 6300 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:19.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.447875 6300 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:19.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.448464 6300 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:19.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.480283 6300 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:19.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.480587 6300 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.480784 6300 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.480968 6300 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.481159 6300 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.481340 6300 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42903/jwks.json
20:16:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.517791 6300 master_runner.cc:386] Master server non-default flags:
20:16:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/data
20:16:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/wal
20:16:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.189:38103
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42903/jwks.json
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/data/info.pb
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.189
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/logs
20:16:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.519201 6300 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.521415 6300 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:19.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.538442 6309 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:19.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.544831 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } attempt: 4
20:16:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.549438 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.189:38103: Network error: Client connection negotiation failed: client connection to 127.6.18.189:38103: connect: Connection refused (error 111)
20:16:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.848875 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } attempt: 5
20:16:19.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.853060 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.189:38103: Network error: Client connection negotiation failed: client connection to 127.6.18.189:38103: connect: Connection refused (error 111)
20:16:20.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.400702 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } attempt: 6
20:16:20.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:20.408250 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.189:38103: Network error: Client connection negotiation failed: client connection to 127.6.18.189:38103: connect: Connection refused (error 111)
20:16:20.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.538882 6312 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:19.539973 6300 server_base.cc:1034] running on GCE node
20:16:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:19.539463 6310 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:20.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.650427 6300 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:20.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.653120 6300 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.655333 6300 hybrid_clock.cc:648] HybridClock initialized: now 1714508180655305 us; error 45 us; skew 500 ppm
20:16:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.656443 6300 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.663252 6300 webserver.cc:460] Webserver started at http://127.6.18.189:43203/ using document root <none> and password file <none>
20:16:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.664343 6300 fs_manager.cc:359] Metadata directory not provided
20:16:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.664630 6300 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.665220 6300 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.670599 6300 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/data/instance:
20:16:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ab0c9972d7549f4902cef31c9bcd5ea"
20:16:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:20 on dist-test-slave-7k6f"
20:16:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.672170 6300 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/wal/instance:
20:16:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ab0c9972d7549f4902cef31c9bcd5ea"
20:16:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:20 on dist-test-slave-7k6f"
20:16:20.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.679618 6300 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
20:16:20.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.685168 6321 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.686230 6300 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
20:16:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.686585 6300 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/wal
20:16:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ab0c9972d7549f4902cef31c9bcd5ea"
20:16:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:20 on dist-test-slave-7k6f"
20:16:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.686986 6300 fs_report.cc:383] FS layout report
20:16:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/wal
20:16:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/wal
20:16:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/data/data
20:16:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:20.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.759845 6300 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:20.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.761459 6300 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:20.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.761992 6300 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:20.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.831064 6300 rpc_server.cc:307] RPC server started. Bound to: 127.6.18.189:38103
20:16:20.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.831167 6372 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.18.189:38103 every 8 connection(s)
20:16:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.833977 6300 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/data/info.pb
20:16:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.839429 6373 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:16:20.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.843595 6218 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 6300
20:16:20.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.844066 6218 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-1/wal/instance
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.847167 6218 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/wal
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/data
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:20.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/logs
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/data/info.pb
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.188:45937
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.188
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42903/jwks.json with env {}
20:16:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.854952 6373 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } has no permanent_uuid. Determining permanent_uuid...
20:16:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.876667 6373 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } has no permanent_uuid. Determining permanent_uuid...
20:16:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.891296 6373 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } has no permanent_uuid. Determining permanent_uuid...
20:16:20.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:20.893543 6322 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:20.897581 6373 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:20.947180 6373 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 1
20:16:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:20.951094 6373 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.019613 6373 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 2
20:16:21.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.023630 6373 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.115090 6373 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 3
20:16:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.118993 6373 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.174985 6375 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.175565 6375 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:21.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.176110 6375 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.206023 6375 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.206323 6375 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.206512 6375 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.206717 6375 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.206952 6375 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.207119 6375 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42903/jwks.json
20:16:21.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.241425 6375 master_runner.cc:386] Master server non-default flags:
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/data
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/wal
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.188:45937
20:16:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42903/jwks.json
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/data/info.pb
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.188
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/logs
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.242676 6375 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.244756 6375 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.262455 6373 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 4
20:16:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.262797 6386 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:21.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.266814 6373 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.468924 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } attempt: 7
20:16:21.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.481637 6298 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } has no permanent_uuid. Determining permanent_uuid...
20:16:21.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.487681 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.546334 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 1
20:16:21.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.550470 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.566193 6373 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 5
20:16:21.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.571833 6373 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.632107 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 2
20:16:21.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.637955 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.724650 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 3
20:16:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.730394 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.870847 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 4
20:16:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.874907 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.119217 6373 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 6
20:16:22.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:22.125705 6373 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.158583 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 5
20:16:22.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:22.163297 6298 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.18.188:45937: Network error: Client connection negotiation failed: client connection to 127.6.18.188:45937: connect: Connection refused (error 111)
20:16:22.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.264043 6388 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:22.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:21.263407 6385 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:22.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:21.265086 6375 server_base.cc:1034] running on GCE node
20:16:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.392665 6375 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.395407 6375 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:22.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.396831 6375 hybrid_clock.cc:648] HybridClock initialized: now 1714508182396777 us; error 57 us; skew 500 ppm
20:16:22.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.397835 6375 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.404188 6375 webserver.cc:460] Webserver started at http://127.6.18.188:43175/ using document root <none> and password file <none>
20:16:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.405193 6375 fs_manager.cc:359] Metadata directory not provided
20:16:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.405417 6375 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:22.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.405974 6375 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.411127 6375 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/data/instance:
20:16:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed5b9eab49044204901e654270bd00b2"
20:16:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:22 on dist-test-slave-7k6f"
20:16:22.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.412559 6375 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/wal/instance:
20:16:22.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed5b9eab49044204901e654270bd00b2"
20:16:22.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:22 on dist-test-slave-7k6f"
20:16:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.420711 6375 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
20:16:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.426831 6400 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.428051 6375 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:16:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.428413 6375 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/wal
20:16:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed5b9eab49044204901e654270bd00b2"
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:22 on dist-test-slave-7k6f"
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.428820 6375 fs_report.cc:383] FS layout report
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/wal
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/wal
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/data/data
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:22.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.476003 6375 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.477609 6375 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.478125 6375 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.547686 6375 rpc_server.cc:307] RPC server started. Bound to: 127.6.18.188:45937
20:16:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.547798 6451 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.18.188:45937 every 8 connection(s)
20:16:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.550567 6375 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/data/info.pb
20:16:22.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.555756 6452 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:16:22.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.560880 6218 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 6375
20:16:22.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.561323 6218 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/master-2/wal/instance
20:16:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.567993 6452 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } has no permanent_uuid. Determining permanent_uuid...
20:16:22.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.604292 6452 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } has no permanent_uuid. Determining permanent_uuid...
20:16:22.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.614239 6452 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } has no permanent_uuid. Determining permanent_uuid...
20:16:22.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.630309 6452 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: Bootstrap starting.
20:16:22.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.637017 6452 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: Neither blocks nor log segments found. Creating new log.
20:16:22.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.639075 6452 log.cc:824] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: Log is configured to *not* fsync() on all Append() calls
20:16:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.643862 6452 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: No bootstrap required, opened a new log
20:16:22.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.666857 6452 raft_consensus.cc:348] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:22.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.667513 6452 raft_consensus.cc:374] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:16:22.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.667709 6452 raft_consensus.cc:729] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ed5b9eab49044204901e654270bd00b2, State: Initialized, Role: FOLLOWER
20:16:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.668383 6452 consensus_queue.cc:260] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:22.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.671178 6461 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.671855 6461 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [sys.catalog]: This master's current role is: FOLLOWER
20:16:22.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.673231 6452 sys_catalog.cc:564] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [sys.catalog]: configured and running, proceeding with master startup.
20:16:22.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:22.701397 6472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:22.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:22.701781 6472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:22.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.715929 6298 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 6
20:16:22.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:22.735103 6406 tablet.cc:2369] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:16:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.697039 6375 master_runner.cc:186] Error getting master registration for 127.6.18.190:33169: OK, instance_id { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" instance_seqno: 1714508179106261 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:16:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.740669 6375 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:16:22.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.751336 6298 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: Bootstrap starting.
20:16:22.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.758671 6298 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: Neither blocks nor log segments found. Creating new log.
20:16:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.761107 6298 log.cc:824] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: Log is configured to *not* fsync() on all Append() calls
20:16:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.766528 6298 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: No bootstrap required, opened a new log
20:16:22.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.798157 6298 raft_consensus.cc:348] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:22.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.799125 6298 raft_consensus.cc:374] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:16:22.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.799427 6298 raft_consensus.cc:729] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 29cf8cd6a10f4dd398c8c4f4fe2a7a7a, State: Initialized, Role: FOLLOWER
20:16:22.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.800376 6298 consensus_queue.cc:260] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.803277 6476 sys_catalog.cc:455] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:22.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.803926 6476 sys_catalog.cc:458] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: This master's current role is: FOLLOWER
20:16:22.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.805400 6298 sys_catalog.cc:564] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: configured and running, proceeding with master startup.
20:16:22.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:22.815850 6252 tablet.cc:2369] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:16:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:22.858846 6487 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:22.859666 6487 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:22.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.838075 6230 master_runner.cc:186] Error getting master registration for 127.6.18.189:38103: OK, instance_id { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" instance_seqno: 1714508180811799 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:16:22.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.873757 6230 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:16:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.895399 6476 raft_consensus.cc:482] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:16:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.895895 6476 raft_consensus.cc:504] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:22.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.898240 6476 leader_election.cc:288] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7ab0c9972d7549f4902cef31c9bcd5ea (127.6.18.189:38103), ed5b9eab49044204901e654270bd00b2 (127.6.18.188:45937)
20:16:22.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.900163 6427 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed5b9eab49044204901e654270bd00b2" is_pre_election: true
20:16:22.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.901270 6427 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 29cf8cd6a10f4dd398c8c4f4fe2a7a7a in term 0.
20:16:22.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.902681 6247 leader_election.cc:302] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 29cf8cd6a10f4dd398c8c4f4fe2a7a7a, ed5b9eab49044204901e654270bd00b2; no voters:
20:16:22.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.903628 6476 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 0 FOLLOWER]: Leader pre-election won for term 1
20:16:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.903970 6476 raft_consensus.cc:482] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:16:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.904311 6476 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 0 FOLLOWER]: Advancing to term 1
20:16:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.910902 6476 raft_consensus.cc:504] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.923986 6476 leader_election.cc:288] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [CANDIDATE]: Term 1 election: Requested vote from peers 7ab0c9972d7549f4902cef31c9bcd5ea (127.6.18.189:38103), ed5b9eab49044204901e654270bd00b2 (127.6.18.188:45937)
20:16:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.924994 6427 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed5b9eab49044204901e654270bd00b2"
20:16:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.925601 6427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [term 0 FOLLOWER]: Advancing to term 1
20:16:22.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.932802 6427 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 29cf8cd6a10f4dd398c8c4f4fe2a7a7a in term 1.
20:16:22.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.934530 6247 leader_election.cc:302] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 29cf8cd6a10f4dd398c8c4f4fe2a7a7a, ed5b9eab49044204901e654270bd00b2; no voters:
20:16:22.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.935608 6476 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 1 FOLLOWER]: Leader election won for term 1
20:16:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.941622 6476 raft_consensus.cc:686] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [term 1 LEADER]: Becoming Leader. State: Replica: 29cf8cd6a10f4dd398c8c4f4fe2a7a7a, State: Running, Role: LEADER
20:16:22.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.942860 6476 consensus_queue.cc:237] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.955088 6489 sys_catalog.cc:455] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 29cf8cd6a10f4dd398c8c4f4fe2a7a7a. Latest consensus state: current_term: 1 leader_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.955999 6489 sys_catalog.cc:458] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: This master's current role is: LEADER
20:16:22.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.965130 6491 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:16:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.972515 6491 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:16:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:22.998812 6427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [term 1 FOLLOWER]: Refusing update from remote peer 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:16:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.000574 6489 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [LEADER]: Connected to new peer: Peer: permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:16:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.035619 6492 mvcc.cc:205] Tried to move back new op lower bound from 7022625517546397696 to 7022625517358837760. Current Snapshot: MvccSnapshot[applied={T|T < 7022625517546397696}]
20:16:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.034114 6461 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 29cf8cd6a10f4dd398c8c4f4fe2a7a7a. Latest consensus state: current_term: 1 leader_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.040185 6461 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [sys.catalog]: This master's current role is: FOLLOWER
20:16:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.044473 6476 sys_catalog.cc:455] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.045280 6476 sys_catalog.cc:458] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: This master's current role is: LEADER
20:16:23.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.057195 6491 catalog_manager.cc:1347] Generated new cluster ID: d9cde7ab5b274939be2c195a773cb157
20:16:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.057518 6491 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:16:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.065958 6461 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.066737 6461 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2 [sys.catalog]: This master's current role is: FOLLOWER
20:16:23.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.106416 6491 catalog_manager.cc:1370] Generated new certificate authority record
20:16:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.113518 6491 catalog_manager.cc:1504] Loading token signing keys...
20:16:23.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.166710 6491 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: Generated new TSK 0
20:16:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.167882 6491 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:16:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.186054 6373 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } attempt: 7
20:16:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.213003 6218 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/wal
20:16:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/data
20:16:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/logs
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/data/info.pb
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.129:0
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.18.129
20:16:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.129
20:16:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:16:23.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.225129 6373 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea: Bootstrap starting.
20:16:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.233028 6373 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea: Neither blocks nor log segments found. Creating new log.
20:16:23.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.236120 6373 log.cc:824] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea: Log is configured to *not* fsync() on all Append() calls
20:16:23.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.268704 6373 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea: No bootstrap required, opened a new log
20:16:23.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.424929 6373 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:23.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.427887 6373 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:16:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.429033 6373 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7ab0c9972d7549f4902cef31c9bcd5ea, State: Initialized, Role: FOLLOWER
20:16:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.431933 6373 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } }
20:16:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.459233 6348 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" is_pre_election: true
20:16:23.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.462596 6346 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [term 0 FOLLOWER]: Advancing to term 1
20:16:23.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.461205 6347 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea"
20:16:23.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.475612 6346 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [term 1 FOLLOWER]: Refusing update from remote peer 29cf8cd6a10f4dd398c8c4f4fe2a7a7a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:16:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.478005 6489 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:16:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.458089 6501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:23.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.488947 6501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [sys.catalog]: This master's current role is: FOLLOWER
20:16:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.486940 6373 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [sys.catalog]: configured and running, proceeding with master startup.
20:16:23.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.529228 6504 mvcc.cc:205] Tried to move back new op lower bound from 7022625517546397696 to 7022625517358837760. Current Snapshot: MvccSnapshot[applied={T|T < 7022625517546397696}]
20:16:23.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.565634 6502 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.568485 6502 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [sys.catalog]: This master's current role is: FOLLOWER
20:16:23.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.567349 6501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [sys.catalog]: SysCatalogTable state changed. Reason: New leader 29cf8cd6a10f4dd398c8c4f4fe2a7a7a. Latest consensus state: current_term: 1 leader_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:23.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.570178 6501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea [sys.catalog]: This master's current role is: FOLLOWER
20:16:23.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.655879 6517 catalog_manager.cc:1259] Loaded cluster ID: d9cde7ab5b274939be2c195a773cb157
20:16:23.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.657480 6517 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea: loading cluster ID for follower catalog manager: success
20:16:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.673641 6517 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea: acquiring CA information for follower catalog manager: success
20:16:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.688886 6517 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 7ab0c9972d7549f4902cef31c9bcd5ea: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:16:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.707684 6472 catalog_manager.cc:1259] Loaded cluster ID: d9cde7ab5b274939be2c195a773cb157
20:16:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.708448 6472 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: loading cluster ID for follower catalog manager: success
20:16:23.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.716519 6472 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: acquiring CA information for follower catalog manager: success
20:16:23.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.734827 6472 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ed5b9eab49044204901e654270bd00b2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:16:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:23.797857 6499 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:23.798563 6499 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:23.799414 6499 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:23.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:23.861181 6499 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:23.862824 6499 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.18.129
20:16:23.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.913268 6489 sys_catalog.cc:455] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "29cf8cd6a10f4dd398c8c4f4fe2a7a7a" member_type: VOTER last_known_addr { host: "127.6.18.190" port: 33169 } } peers { permanent_uuid: "7ab0c9972d7549f4902cef31c9bcd5ea" member_type: VOTER last_known_addr { host: "127.6.18.189" port: 38103 } } peers { permanent_uuid: "ed5b9eab49044204901e654270bd00b2" member_type: VOTER last_known_addr { host: "127.6.18.188" port: 45937 } } }
20:16:23.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.915280 6489 sys_catalog.cc:458] T 00000000000000000000000000000000 P 29cf8cd6a10f4dd398c8c4f4fe2a7a7a [sys.catalog]: This master's current role is: LEADER
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.933816 6499 tablet_server_runner.cc:78] Tablet server non-default flags:
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/data
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/wal
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.129:0
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/data/info.pb
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.129
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.18.129
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/logs
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:23.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.935562 6499 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:23.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:23.937783 6499 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:23.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:23.980733 6528 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:23.990175 6531 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:23.984522 6527 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.588299 6499 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:16:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:25.588063 6529 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:16:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.593364 6499 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.596453 6499 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.597934 6499 hybrid_clock.cc:648] HybridClock initialized: now 1714508185598011 us; error 185 us; skew 500 ppm
20:16:25.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.598991 6499 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.607676 6499 webserver.cc:460] Webserver started at http://127.6.18.129:45473/ using document root <none> and password file <none>
20:16:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.608932 6499 fs_manager.cc:359] Metadata directory not provided
20:16:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.609234 6499 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.609773 6499 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.617147 6499 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/data/instance:
20:16:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b905f8c92ee14680b8c90480a77c19a9"
20:16:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:25 on dist-test-slave-7k6f"
20:16:25.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.618676 6499 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/wal/instance:
20:16:25.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b905f8c92ee14680b8c90480a77c19a9"
20:16:25.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:25 on dist-test-slave-7k6f"
20:16:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.628371 6499 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.008s sys 0.000s
20:16:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.636242 6537 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.637717 6499 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
20:16:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.638154 6499 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/wal
20:16:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b905f8c92ee14680b8c90480a77c19a9"
20:16:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:25 on dist-test-slave-7k6f"
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.638659 6499 fs_report.cc:383] FS layout report
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/wal
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/wal
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/data/data
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.709458 6499 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.711494 6499 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.712080 6499 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:25.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.715518 6499 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:16:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.721318 6499 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:16:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.721634 6499 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:16:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.721940 6499 ts_tablet_manager.cc:590] Registered 0 tablets
20:16:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.722162 6499 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:16:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.906672 6499 rpc_server.cc:307] RPC server started. Bound to: 127.6.18.129:37007
20:16:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.906819 6649 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.18.129:37007 every 8 connection(s)
20:16:25.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.909425 6499 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/data/info.pb
20:16:25.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.917843 6218 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 6499
20:16:25.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.918356 6218 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-0/wal/instance
20:16:25.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.942142 6218 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:25.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:25.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/wal
20:16:25.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/data
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/logs
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/data/info.pb
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:25.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.130:0
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.18.130
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.130
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:25.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:16:25.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.980352 6652 heartbeater.cc:347] Connected to a master server at 127.6.18.190:33169
20:16:25.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.980870 6652 heartbeater.cc:464] Registering TS with master...
20:16:25.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.982275 6652 heartbeater.cc:511] Master 127.6.18.190:33169 requested a full tablet report, sending...
20:16:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.985432 6263 ts_manager.cc:194] Registered new tserver with Master: b905f8c92ee14680b8c90480a77c19a9 (127.6.18.129:37007)
20:16:25.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.988067 6263 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.18.129:47493
20:16:25.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.989718 6650 heartbeater.cc:347] Connected to a master server at 127.6.18.189:38103
20:16:25.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.990073 6650 heartbeater.cc:464] Registering TS with master...
20:16:25.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.990959 6650 heartbeater.cc:511] Master 127.6.18.189:38103 requested a full tablet report, sending...
20:16:25.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.993983 6338 ts_manager.cc:194] Registered new tserver with Master: b905f8c92ee14680b8c90480a77c19a9 (127.6.18.129:37007)
20:16:25.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.994467 6651 heartbeater.cc:347] Connected to a master server at 127.6.18.188:45937
20:16:25.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.994805 6651 heartbeater.cc:464] Registering TS with master...
20:16:25.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.995453 6651 heartbeater.cc:511] Master 127.6.18.188:45937 requested a full tablet report, sending...
20:16:25.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:25.998490 6417 ts_manager.cc:194] Registered new tserver with Master: b905f8c92ee14680b8c90480a77c19a9 (127.6.18.129:37007)
20:16:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:26.297864 6658 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:26.298344 6658 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:26.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:26.298822 6658 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:26.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:26.329501 6658 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:26.330358 6658 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.18.130
20:16:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:26.364216 6658 tablet_server_runner.cc:78] Tablet server non-default flags:
20:16:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/data
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/wal
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.130:0
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/data/info.pb
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.130
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.18.130
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/logs
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:16:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:26.365564 6658 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:26.367703 6658 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:26.389401 6666 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:26.994436 6652 heartbeater.cc:503] Master 127.6.18.190:33169 was elected leader, sending a full tablet report...
20:16:27.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:26.390015 6667 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:27.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:26.390743 6669 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:27.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:26.391047 6658 server_base.cc:1034] running on GCE node
20:16:27.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.510468 6658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:27.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.513715 6658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:27.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.515177 6658 hybrid_clock.cc:648] HybridClock initialized: now 1714508187515150 us; error 63 us; skew 500 ppm
20:16:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.515952 6658 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:27.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.522153 6658 webserver.cc:460] Webserver started at http://127.6.18.130:35261/ using document root <none> and password file <none>
20:16:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.523204 6658 fs_manager.cc:359] Metadata directory not provided
20:16:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.523437 6658 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.523854 6658 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.528704 6658 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/data/instance:
20:16:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe2923693b349d485a18978b411c966"
20:16:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:27 on dist-test-slave-7k6f"
20:16:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.529767 6658 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/wal/instance:
20:16:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe2923693b349d485a18978b411c966"
20:16:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:27 on dist-test-slave-7k6f"
20:16:27.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.537837 6658 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
20:16:27.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.543473 6677 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.544562 6658 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
20:16:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.544833 6658 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/wal
20:16:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fe2923693b349d485a18978b411c966"
20:16:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:27 on dist-test-slave-7k6f"
20:16:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.545195 6658 fs_report.cc:383] FS layout report
20:16:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/wal
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/wal
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/data/data
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.619251 6658 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:27.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.620709 6658 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:27.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.621131 6658 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.623759 6658 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:16:27.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.627830 6658 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:16:27.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.628026 6658 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:16:27.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.628227 6658 ts_tablet_manager.cc:590] Registered 0 tablets
20:16:27.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.628366 6658 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:16:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.765869 6658 rpc_server.cc:307] RPC server started. Bound to: 127.6.18.130:36803
20:16:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.766043 6790 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.18.130:36803 every 8 connection(s)
20:16:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.768544 6658 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/data/info.pb
20:16:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.775713 6218 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 6658
20:16:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.776456 6218 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-1/wal/instance
20:16:27.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.796388 6218 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:27.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/wal
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/data
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/logs
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/data/info.pb
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.131:0
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.18.131
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.131
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:16:27.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.837743 6792 heartbeater.cc:347] Connected to a master server at 127.6.18.188:45937
20:16:27.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.838100 6791 heartbeater.cc:347] Connected to a master server at 127.6.18.189:38103
20:16:27.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.838246 6792 heartbeater.cc:464] Registering TS with master...
20:16:27.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.838532 6791 heartbeater.cc:464] Registering TS with master...
20:16:27.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.839620 6792 heartbeater.cc:511] Master 127.6.18.188:45937 requested a full tablet report, sending...
20:16:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.839756 6791 heartbeater.cc:511] Master 127.6.18.189:38103 requested a full tablet report, sending...
20:16:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.842752 6417 ts_manager.cc:194] Registered new tserver with Master: 6fe2923693b349d485a18978b411c966 (127.6.18.130:36803)
20:16:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.842752 6338 ts_manager.cc:194] Registered new tserver with Master: 6fe2923693b349d485a18978b411c966 (127.6.18.130:36803)
20:16:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.849673 6793 heartbeater.cc:347] Connected to a master server at 127.6.18.190:33169
20:16:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.850030 6793 heartbeater.cc:464] Registering TS with master...
20:16:27.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.850950 6793 heartbeater.cc:511] Master 127.6.18.190:33169 requested a full tablet report, sending...
20:16:27.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.853268 6263 ts_manager.cc:194] Registered new tserver with Master: 6fe2923693b349d485a18978b411c966 (127.6.18.130:36803)
20:16:27.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:27.855376 6263 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.18.130:58263
20:16:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:28.177631 6801 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:28.178107 6801 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:28.178553 6801 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:28.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:28.209177 6801 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:28.209978 6801 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.18.131
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:28.243664 6801 tablet_server_runner.cc:78] Tablet server non-default flags:
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.18.148:35457
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/data
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/wal
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.18.131:0
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/data/info.pb
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.18.131
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.18.190:33169,127.6.18.189:38103,127.6.18.188:45937
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.18.131
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/logs
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:16:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:28.244894 6801 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:28.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:28.246798 6801 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:28.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:28.264951 6807 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:28.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:28.858994 6793 heartbeater.cc:503] Master 127.6.18.190:33169 was elected leader, sending a full tablet report...
20:16:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:28.265861 6810 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:28.265357 6808 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:28.266418 6801 server_base.cc:1034] running on GCE node
20:16:29.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.375777 6801 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:29.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.378077 6801 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:29.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.379441 6801 hybrid_clock.cc:648] HybridClock initialized: now 1714508189379393 us; error 64 us; skew 500 ppm
20:16:29.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.380240 6801 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:29.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.390892 6801 webserver.cc:460] Webserver started at http://127.6.18.131:42889/ using document root <none> and password file <none>
20:16:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.391837 6801 fs_manager.cc:359] Metadata directory not provided
20:16:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.392047 6801 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.392439 6801 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.397223 6801 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/data/instance:
20:16:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c98c2a24b144568bcf8097af39181b1"
20:16:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:29 on dist-test-slave-7k6f"
20:16:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.398283 6801 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/wal/instance:
20:16:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c98c2a24b144568bcf8097af39181b1"
20:16:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:29 on dist-test-slave-7k6f"
20:16:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.405297 6801 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
20:16:29.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.410739 6817 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.411831 6801 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.412130 6801 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/wal
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c98c2a24b144568bcf8097af39181b1"
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:29 on dist-test-slave-7k6f"
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.412433 6801 fs_report.cc:383] FS layout report
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/wal
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/wal
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/data/data
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:29.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.471980 6801 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:29.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.473416 6801 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:29.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.473922 6801 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:29.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.476495 6801 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:16:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.480638 6801 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:16:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.480830 6801 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:16:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.481027 6801 ts_tablet_manager.cc:590] Registered 0 tablets
20:16:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.481165 6801 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:16:29.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.627560 6801 rpc_server.cc:307] RPC server started. Bound to: 127.6.18.131:40801
20:16:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.627687 6930 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.18.131:40801 every 8 connection(s)
20:16:29.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.630252 6801 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/data/info.pb
20:16:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.641050 6218 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 6801
20:16:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.641671 6218 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster1529399732861790215/ts-2/wal/instance
20:16:29.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.689745 6933 heartbeater.cc:347] Connected to a master server at 127.6.18.190:33169
20:16:29.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.690199 6933 heartbeater.cc:464] Registering TS with master...
20:16:29.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.691403 6933 heartbeater.cc:511] Master 127.6.18.190:33169 requested a full tablet report, sending...
20:16:29.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.691603 6931 heartbeater.cc:347] Connected to a master server at 127.6.18.189:38103
20:16:29.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.691952 6931 heartbeater.cc:464] Registering TS with master...
20:16:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.692766 6931 heartbeater.cc:511] Master 127.6.18.189:38103 requested a full tablet report, sending...
20:16:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.693220 6932 heartbeater.cc:347] Connected to a master server at 127.6.18.188:45937
20:16:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.693459 6932 heartbeater.cc:464] Registering TS with master...
20:16:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.694240 6263 ts_manager.cc:194] Registered new tserver with Master: 1c98c2a24b144568bcf8097af39181b1 (127.6.18.131:40801)
20:16:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.694341 6932 heartbeater.cc:511] Master 127.6.18.188:45937 requested a full tablet report, sending...
20:16:29.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.695770 6338 ts_manager.cc:194] Registered new tserver with Master: 1c98c2a24b144568bcf8097af39181b1 (127.6.18.131:40801)
20:16:29.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.696084 6263 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.18.131:46265
20:16:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.696982 6417 ts_manager.cc:194] Registered new tserver with Master: 1c98c2a24b144568bcf8097af39181b1 (127.6.18.131:40801)
20:16:29.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:29.707304 6218 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:16:29.718 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:16:29.719 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:16:29.730 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.18.190"
port: 33169
}
bound_http_address {
host: "127.6.18.190"
port: 36847
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.18.189"
port: 38103
}
bound_http_address {
host: "127.6.18.189"
port: 43203
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.18.188"
port: 45937
}
bound_http_address {
host: "127.6.18.188"
port: 43175
}
}
}
20:16:29.735 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:16:29.737 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.18.129"
port: 37007
}
bound_http_address {
host: "127.6.18.129"
port: 45473
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.18.130"
port: 36803
}
bound_http_address {
host: "127.6.18.130"
port: 35261
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.18.131"
port: 40801
}
bound_http_address {
host: "127.6.18.131"
port: 42889
}
}
}
20:16:29.737 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:16:29.915 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:16:30.035 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskASVjAM/test-tmp
20:16:30.036 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:16:30.036 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskASVjAM/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:16:30.039 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:16:30.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:30.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:16:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:30.433519 6942 mini_oidc.cc:152] Starting JWKS server
20:16:30.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:30.437006 6942 webserver.cc:460] Webserver started at http://127.0.0.1:42949/ using document root <none> and password file <none>
20:16:30.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:30.437455 6942 mini_oidc.cc:189] Starting OIDC Discovery server
20:16:30.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:30.439146 6942 webserver.cc:460] Webserver started at http://127.0.0.1:35519/ using document root <none> and password file <none>
20:16:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:16:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/principal' for realm 'KRBTEST.COM',
20:16:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:16:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:30 dist-test-slave-7k6f krb5kdc[6955](info): setting up network...
20:16:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:16:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:30 dist-test-slave-7k6f krb5kdc[6955](info): set up 2 sockets
20:16:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:30 dist-test-slave-7k6f krb5kdc[6955](info): commencing operation
20:16:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:16:30.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:30.699828 6933 heartbeater.cc:503] Master 127.6.18.190:33169 was elected leader, sending a full tablet report...
20:16:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:31.476436 6942 mini_kdc.cc:121] Time spent starting KDC: real 0.999s user 0.011s sys 0.008s
20:16:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:16:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:16:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:16:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:16:31.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:16:31.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:16:31.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:16:31.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:16:31.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:16:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:16:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/test-user.keytab.
20:16:31.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/test-user.keytab.
20:16:31.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:31 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508191, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:31.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:16:31.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T20:16:31Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:16:31.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T20:16:31Z Disabled control of system clock
20:16:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.199.190@KRBTEST.COM; defaulting to no policy
20:16:31.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:16:31.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.199.190@KRBTEST.COM" created.
20:16:31.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:16:31.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.199.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.190.keytab.
20:16:31.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.199.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.190.keytab.
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:31.672016 6942 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/logs
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/info.pb
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.190:39305
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.190
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.190.keytab
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.190
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:16:31.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:31.990036 6969 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:31.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:31.990731 6969 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:31.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:31.991437 6969 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.024430 6969 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:16:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.024709 6969 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:32.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.025115 6969 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:32.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.025432 6969 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:32.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.025735 6969 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.026039 6969 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.026258 6969 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42949/jwks.json
20:16:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:32.062814 6969 master_runner.cc:386] Master server non-default flags:
20:16:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.190.keytab
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.190
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.190:39305
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/info.pb
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.190
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/logs
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:32.064252 6969 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:32.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:32.065712 6969 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.082273 6975 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.082484 6976 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:32.083552 6978 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:33.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:32.085337 6969 server_base.cc:1034] running on GCE node
20:16:33.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.235100 6969 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.237763 6969 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.239148 6969 hybrid_clock.cc:648] HybridClock initialized: now 1714508193239114 us; error 42 us; skew 500 ppm
20:16:33.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.240067 6969 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:33.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:33 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508193, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:33.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.246184 6969 init.cc:377] Logged in from keytab as kudu/127.6.199.190@KRBTEST.COM (short username kudu)
20:16:33.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.251452 6969 webserver.cc:460] Webserver started at http://127.6.199.190:40957/ using document root <none> and password file <none>
20:16:33.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.252419 6969 fs_manager.cc:359] Metadata directory not provided
20:16:33.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.252646 6969 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:33.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.253165 6969 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:33.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.258141 6969 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/instance:
20:16:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d29e5163e05447b7a40f3396092bec67"
20:16:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:33 on dist-test-slave-7k6f"
20:16:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.259543 6969 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal/instance:
20:16:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d29e5163e05447b7a40f3396092bec67"
20:16:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:33 on dist-test-slave-7k6f"
20:16:33.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.267068 6969 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
20:16:33.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.272859 6987 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.273995 6969 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:16:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.274331 6969 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d29e5163e05447b7a40f3396092bec67"
20:16:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:33 on dist-test-slave-7k6f"
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.274744 6969 fs_report.cc:383] FS layout report
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/data
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:33.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.336004 6969 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:33.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.341696 6969 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:33.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.342116 6969 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:33.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.410681 6969 rpc_server.cc:307] RPC server started. Bound to: 127.6.199.190:39305
20:16:33.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.410851 7038 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.199.190:39305 every 8 connection(s)
20:16:33.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.413491 6969 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/info.pb
20:16:33.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.415297 6942 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 6969
20:16:33.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.415938 6942 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal/instance
20:16:33.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.419706 7039 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:16:33.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.199.189@KRBTEST.COM; defaulting to no policy
20:16:33.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.431080 7039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } has no permanent_uuid. Determining permanent_uuid...
20:16:33.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:16:33.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.199.189@KRBTEST.COM" created.
20:16:33.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:33 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508193, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.190@KRBTEST.COM for kudu/127.6.199.190@KRBTEST.COM
20:16:33.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:16:33.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.199.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.189.keytab.
20:16:33.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.199.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.189.keytab.
20:16:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.463775 6942 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/logs
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/info.pb
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.189:42517
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.189
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.189.keytab
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.189
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:16:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.471854 7039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } has no permanent_uuid. Determining permanent_uuid...
20:16:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.474779 6988 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:33.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.479537 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.189:42517: Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:33.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.529098 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } attempt: 1
20:16:33.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.533327 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.189:42517: Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.601807 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } attempt: 2
20:16:33.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.606184 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.189:42517: Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.697566 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } attempt: 3
20:16:33.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.701453 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.189:42517: Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:33.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.792258 7045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.792923 7045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.793622 7045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.825937 7045 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:16:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.826234 7045 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.826664 7045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.826987 7045 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.827287 7045 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.827586 7045 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.827802 7045 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42949/jwks.json
20:16:33.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.844837 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } attempt: 4
20:16:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.849581 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.189:42517: Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.868654 7045 master_runner.cc:386] Master server non-default flags:
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.189.keytab
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.189
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.189:42517
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/info.pb
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.189
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/logs
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:33.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:33.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.869848 7045 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:33.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.870925 7045 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:33.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.888298 7054 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:34.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:34.149053 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } attempt: 5
20:16:34.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:34.152812 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.189:42517: Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:34.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:34.700512 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } attempt: 6
20:16:34.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:34.709915 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.189:42517: Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:35.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.888823 7053 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:35.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:33.889783 7045 server_base.cc:1034] running on GCE node
20:16:35.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:33.889336 7056 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:35.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.033373 7045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.036273 7045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:35.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.037637 7045 hybrid_clock.cc:648] HybridClock initialized: now 1714508195037609 us; error 40 us; skew 500 ppm
20:16:35.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.038421 7045 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:35.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:35 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508195, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:35.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.050504 7045 init.cc:377] Logged in from keytab as kudu/127.6.199.189@KRBTEST.COM (short username kudu)
20:16:35.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.055742 7045 webserver.cc:460] Webserver started at http://127.6.199.189:41503/ using document root <none> and password file <none>
20:16:35.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.056757 7045 fs_manager.cc:359] Metadata directory not provided
20:16:35.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.056983 7045 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:35.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.057485 7045 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:35.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.062870 7045 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/instance:
20:16:35.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3098cb31b4e3430b8994ba9509a1ce7b"
20:16:35.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:35 on dist-test-slave-7k6f"
20:16:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.064154 7045 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal/instance:
20:16:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3098cb31b4e3430b8994ba9509a1ce7b"
20:16:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:35 on dist-test-slave-7k6f"
20:16:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.072147 7045 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
20:16:35.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.077817 7066 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:35.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.078923 7045 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
20:16:35.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.079325 7045 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:35.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3098cb31b4e3430b8994ba9509a1ce7b"
20:16:35.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:35 on dist-test-slave-7k6f"
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.079685 7045 fs_report.cc:383] FS layout report
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/data
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:35.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.138875 7045 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:35.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.144676 7045 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:35.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.145210 7045 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:35.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.227834 7045 rpc_server.cc:307] RPC server started. Bound to: 127.6.199.189:42517
20:16:35.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.227923 7117 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.199.189:42517 every 8 connection(s)
20:16:35.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.230587 7045 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/info.pb
20:16:35.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.237010 7118 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:16:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.240474 6942 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 7045
20:16:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.240828 6942 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal/instance
20:16:35.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.250135 7118 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } has no permanent_uuid. Determining permanent_uuid...
20:16:35.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.199.188@KRBTEST.COM; defaulting to no policy
20:16:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:16:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.199.188@KRBTEST.COM" created.
20:16:35.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:35 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508195, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.189@KRBTEST.COM for kudu/127.6.199.190@KRBTEST.COM
20:16:35.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.288035 7118 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } has no permanent_uuid. Determining permanent_uuid...
20:16:35.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:16:35.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.199.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.188.keytab.
20:16:35.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.199.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.188.keytab.
20:16:35.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.293326 6942 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:35.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:35.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:35.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/logs
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/info.pb
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.188:44655
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.188
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.188.keytab
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.188
20:16:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:16:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:35 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508195, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.189@KRBTEST.COM for kudu/127.6.199.189@KRBTEST.COM
20:16:35.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.314751 7118 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } has no permanent_uuid. Determining permanent_uuid...
20:16:35.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.317090 7069 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.321770 7118 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.371230 7118 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 1
20:16:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.375422 7118 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.443832 7118 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 2
20:16:35.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.448549 7118 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.539930 7118 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 3
20:16:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.543550 7118 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.623332 7125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.623878 7125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.624459 7125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.661528 7125 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:16:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.661759 7125 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.662086 7125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.662304 7125 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.662485 7125 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.662748 7125 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.662916 7125 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42949/jwks.json
20:16:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.686993 7118 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 4
20:16:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.691066 7118 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.701704 7125 master_runner.cc:386] Master server non-default flags:
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.188.keytab
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.188
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.188:44655
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/info.pb
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.188
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/logs
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.702880 7125 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:35.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.703891 7125 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.727876 7135 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.770401 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } attempt: 7
20:16:35.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:35 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508193, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.190@KRBTEST.COM for kudu/127.6.199.189@KRBTEST.COM
20:16:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.797734 7039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } has no permanent_uuid. Determining permanent_uuid...
20:16:35.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.805877 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.864404 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 1
20:16:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.869935 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.951309 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 2
20:16:35.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.955065 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.990483 7118 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 5
20:16:35.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.996098 7118 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:36.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.040730 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 3
20:16:36.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:36.046770 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:36.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.187196 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 4
20:16:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:36.192548 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.475998 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 5
20:16:36.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:36.480768 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.543728 7118 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 6
20:16:36.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:36.551501 7118 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.728227 7132 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:35.729699 7125 server_base.cc:1034] running on GCE node
20:16:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:35.729758 7133 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:36.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.882807 7125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.885744 7125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.887125 7125 hybrid_clock.cc:648] HybridClock initialized: now 1714508196887113 us; error 81 us; skew 500 ppm
20:16:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.887948 7125 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:36 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508196, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.894584 7125 init.cc:377] Logged in from keytab as kudu/127.6.199.188@KRBTEST.COM (short username kudu)
20:16:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.900125 7125 webserver.cc:460] Webserver started at http://127.6.199.188:39581/ using document root <none> and password file <none>
20:16:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.901221 7125 fs_manager.cc:359] Metadata directory not provided
20:16:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.901425 7125 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.901831 7125 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:16:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.908978 7125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/instance:
20:16:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79153dd47fc74902ae24f34e1a774830"
20:16:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:36 on dist-test-slave-7k6f"
20:16:36.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.910501 7125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal/instance:
20:16:36.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79153dd47fc74902ae24f34e1a774830"
20:16:36.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:36 on dist-test-slave-7k6f"
20:16:36.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.920024 7125 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
20:16:36.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.926306 7149 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.927479 7125 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.927807 7125 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79153dd47fc74902ae24f34e1a774830"
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:36 on dist-test-slave-7k6f"
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.928126 7125 fs_report.cc:383] FS layout report
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/data
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:36.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:36.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.986968 7125 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.993268 7125 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:36.993701 7125 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.033349 7039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } attempt: 6
20:16:37.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.046460 7039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.199.188:44655: Network error: Client connection negotiation failed: client connection to 127.6.199.188:44655: connect: Connection refused (error 111)
20:16:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.079088 7125 rpc_server.cc:307] RPC server started. Bound to: 127.6.199.188:44655
20:16:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.079159 7202 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.199.188:44655 every 8 connection(s)
20:16:37.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.081840 7125 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/info.pb
20:16:37.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.089392 7203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:16:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.090914 6942 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 7125
20:16:37.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.091338 6942 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal/instance
20:16:37.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.109519 7203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } has no permanent_uuid. Determining permanent_uuid...
20:16:37.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:37 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508191, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.199.188@KRBTEST.COM
20:16:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:37 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508196, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.188@KRBTEST.COM for kudu/127.6.199.190@KRBTEST.COM
20:16:37.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.146605 7203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } has no permanent_uuid. Determining permanent_uuid...
20:16:37.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:37 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508196, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.188@KRBTEST.COM for kudu/127.6.199.189@KRBTEST.COM
20:16:37.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:37 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508191, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.199.189@KRBTEST.COM
20:16:37.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.172822 7203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } has no permanent_uuid. Determining permanent_uuid...
20:16:37.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:37 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508196, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.188@KRBTEST.COM for kudu/127.6.199.188@KRBTEST.COM
20:16:37.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:37 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508191, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.199.190@KRBTEST.COM
20:16:37.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.201808 7203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: Bootstrap starting.
20:16:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.203330 6942 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:16:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.207121 7203 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: Neither blocks nor log segments found. Creating new log.
20:16:37.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.209376 7203 log.cc:824] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: Log is configured to *not* fsync() on all Append() calls
20:16:37.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:37.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:16:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.216307 7203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: No bootstrap required, opened a new log
20:16:37.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:16:37.226 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:16:37.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.199.190"
port: 39305
}
bound_http_address {
host: "127.6.199.190"
port: 40957
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.199.189"
port: 42517
}
bound_http_address {
host: "127.6.199.189"
port: 41503
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.199.188"
port: 44655
}
bound_http_address {
host: "127.6.199.188"
port: 39581
}
}
}
20:16:37.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:16:37.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:37.229 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:16:37.231 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:16:37.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.244076 7203 raft_consensus.cc:348] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.244738 7203 raft_consensus.cc:374] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:16:37.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.245517 7203 raft_consensus.cc:729] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 79153dd47fc74902ae24f34e1a774830, State: Initialized, Role: FOLLOWER
20:16:37.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.246749 7203 consensus_queue.cc:260] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:37.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.251976 7213 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } } }
20:16:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.253755 7203 sys_catalog.cc:564] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [sys.catalog]: configured and running, proceeding with master startup.
20:16:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.253912 7213 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [sys.catalog]: This master's current role is: FOLLOWER
20:16:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:37 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508197, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:16:37.269 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:37.271 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc
20:16:37.278 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:16:37.280 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc
20:16:37.281 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:16:37.285 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:16:37.287 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:16:37.287 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:16:37.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.296350 7225 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:37.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.296705 7225 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.280504 7125 master_runner.cc:186] Error getting master registration for 127.6.199.190:39305: OK, instance_id { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" instance_seqno: 1714508193391436 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:16:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.318331 7125 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:16:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:37 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508197, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:16:37.328 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:37.328 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc
20:16:37.330 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:16:37.330 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.199.189:42517
20:16:37.332 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:16:37.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.332590 6942 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 7045
20:16:37.355 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:37.355 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.199.190:39305
20:16:37.355 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:16:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.356200 6942 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 6969
20:16:37.378 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:37.378 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.199.188:44655
20:16:37.379 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:16:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.379489 6942 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 7125
20:16:37.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:37.402 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.199.189:42517
20:16:37.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.405908 6942 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/logs
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/info.pb
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.189:42517
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.189
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41503
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.189.keytab
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.189
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:16:37.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.713320 7230 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:37.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.714020 7230 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:37.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.714828 7230 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:37.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.748256 7230 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:16:37.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.748558 7230 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.748965 7230 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.749270 7230 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.749564 7230 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:37.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.749850 7230 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:37.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.750066 7230 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42949/jwks.json
20:16:37.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.794301 7230 master_runner.cc:386] Master server non-default flags:
20:16:37.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:37.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:37.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.189.keytab
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.189
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.189:42517
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/info.pb
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.189
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41503
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/logs
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:37.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:37.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:37.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:37.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:37.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.795864 7230 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:37.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.797273 7230 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.813218 7237 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:38.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.813647 7238 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:38.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:37.814152 7240 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:38.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:37.814692 7230 server_base.cc:1034] running on GCE node
20:16:38.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:38.984856 7230 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:38.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:38.988083 7230 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:38.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:38.989620 7230 hybrid_clock.cc:648] HybridClock initialized: now 1714508198989581 us; error 57 us; skew 500 ppm
20:16:38.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:38.990514 7230 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:38.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:38 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508198, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:38.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:38.996590 7230 init.cc:377] Logged in from keytab as kudu/127.6.199.189@KRBTEST.COM (short username kudu)
20:16:39.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.001832 7230 webserver.cc:460] Webserver started at http://127.6.199.189:41503/ using document root <none> and password file <none>
20:16:39.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.002847 7230 fs_manager.cc:359] Metadata directory not provided
20:16:39.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.003119 7230 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.011221 7230 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
20:16:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.015614 7249 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.016824 7230 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
20:16:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.017130 7230 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3098cb31b4e3430b8994ba9509a1ce7b"
20:16:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:35 on dist-test-slave-7k6f"
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.019524 7230 fs_report.cc:383] FS layout report
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/data
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:39.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.073287 7230 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.079344 7230 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.079782 7230 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.155697 7230 rpc_server.cc:307] RPC server started. Bound to: 127.6.199.189:42517
20:16:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.155807 7300 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.199.189:42517 every 8 connection(s)
20:16:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.158376 7230 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/data/info.pb
20:16:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.159448 6942 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 7230
20:16:39.160 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:39.160 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.199.190:39305
20:16:39.161 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.162695 6942 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/logs
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/info.pb
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.190:39305
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.190
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40957
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.190.keytab
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.190
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:39.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:16:39.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.171386 7301 sys_catalog.cc:263] Verifying existing consensus state
20:16:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240430 20:16:39.172452 7301 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:16:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.172991 7230 master.cc:537] Master@127.6.199.189:42517 shutting down...
20:16:39.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.236907 7230 master.cc:552] Master@127.6.199.189:42517 shutdown complete.
20:16:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:16:39.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.487319 7303 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:39.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.487902 7303 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.488552 7303 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:39.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.519405 7303 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:16:39.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.519650 7303 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.519908 7303 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.520109 7303 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.520299 7303 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.520484 7303 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.520637 7303 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42949/jwks.json
20:16:39.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.566324 7303 master_runner.cc:386] Master server non-default flags:
20:16:39.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.190.keytab
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.190
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.190:39305
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:16:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/info.pb
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.190
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40957
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/logs
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.568058 7303 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:39.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:39.569330 7303 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:39.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.595387 7310 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:40.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.595885 7311 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:40.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:39.599574 7313 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:40.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:40.802837 7312 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1203 milliseconds
20:16:40.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.802942 7303 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:16:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.804244 7303 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:40.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.807493 7303 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.808873 7303 hybrid_clock.cc:648] HybridClock initialized: now 1714508200808833 us; error 55 us; skew 500 ppm
20:16:40.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.809823 7303 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:40.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:40 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508200, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:40.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.820658 7303 init.cc:377] Logged in from keytab as kudu/127.6.199.190@KRBTEST.COM (short username kudu)
20:16:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.825723 7303 webserver.cc:460] Webserver started at http://127.6.199.190:40957/ using document root <none> and password file <none>
20:16:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.826591 7303 fs_manager.cc:359] Metadata directory not provided
20:16:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.826854 7303 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.835157 7303 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
20:16:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.840003 7322 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:40.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.841033 7303 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
20:16:40.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.841321 7303 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:40.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d29e5163e05447b7a40f3396092bec67"
20:16:40.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:33 on dist-test-slave-7k6f"
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.843586 7303 fs_report.cc:383] FS layout report
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/data
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:40.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.893646 7303 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.899255 7303 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:40.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.899672 7303 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.970870 7303 rpc_server.cc:307] RPC server started. Bound to: 127.6.199.190:39305
20:16:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.970978 7373 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.199.190:39305 every 8 connection(s)
20:16:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.973661 7303 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/data/info.pb
20:16:40.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:40.978 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.199.188:44655
20:16:40.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:16:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.976505 6942 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 7303
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.980121 6942 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/logs
20:16:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/info.pb
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.188:44655
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.188
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39581
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.188.keytab
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.188
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:16:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.987963 7374 sys_catalog.cc:263] Verifying existing consensus state
20:16:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240430 20:16:40.988802 7374 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:16:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:40.989236 7303 master.cc:537] Master@127.6.199.190:39305 shutting down...
20:16:41.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:41.052542 7303 master.cc:552] Master@127.6.199.190:39305 shutdown complete.
20:16:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:16:41.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.305882 7376 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:16:41.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.306532 7376 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:16:41.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.307202 7376 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:16:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.340312 7376 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:16:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.340546 7376 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:16:41.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.340808 7376 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:16:41.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.341014 7376 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:16:41.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.341205 7376 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:16:41.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.341392 7376 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:16:41.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.341534 7376 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42949/jwks.json
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:41.379580 7376 master_runner.cc:386] Master server non-default flags:
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.199.148:33015
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.199.190:39305,127.6.199.189:42517,127.6.199.188:44655
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/krb5kdc/kudu_127.6.199.188.keytab
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.199.188
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:16:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.199.188:44655
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42949/jwks.json
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/info.pb
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.199.188
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39581
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/logs
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 19:43:22 UTC on 1a15035a6830
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 64
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:16:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:41.380775 7376 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:16:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:41.381963 7376 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:16:41.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.398219 7383 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:42.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.398633 7386 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:42.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:41.401015 7376 server_base.cc:1034] running on GCE node
20:16:42.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:41.399201 7384 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:16:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.538542 7376 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:16:42.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.541157 7376 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:16:42.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.542594 7376 hybrid_clock.cc:648] HybridClock initialized: now 1714508202542543 us; error 54 us; skew 500 ppm
20:16:42.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.543460 7376 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:16:42.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:42 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508202, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:42.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.549505 7376 init.cc:377] Logged in from keytab as kudu/127.6.199.188@KRBTEST.COM (short username kudu)
20:16:42.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.555027 7376 webserver.cc:460] Webserver started at http://127.6.199.188:39581/ using document root <none> and password file <none>
20:16:42.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.555907 7376 fs_manager.cc:359] Metadata directory not provided
20:16:42.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.556116 7376 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:16:42.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.564888 7376 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
20:16:42.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.569479 7394 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:16:42.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.570691 7376 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
20:16:42.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.571045 7376 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data,/tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:42.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79153dd47fc74902ae24f34e1a774830"
20:16:42.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 20:16:36 on dist-test-slave-7k6f"
20:16:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.573443 7376 fs_report.cc:383] FS layout report
20:16:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:16:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/wal
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/data
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:16:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:16:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.634306 7376 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:16:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.641531 7376 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:16:42.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.641990 7376 kserver.cc:163] Server-wide thread pool size limit: 3276
20:16:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.712528 7376 rpc_server.cc:307] RPC server started. Bound to: 127.6.199.188:44655
20:16:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.712651 7445 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.199.188:44655 every 8 connection(s)
20:16:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.715426 7376 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskASVjAM/test-tmp/mini-kudu-cluster146096649020363209/master-2/data/info.pb
20:16:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.716120 6942 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu as pid 7376
20:16:42.717 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:16:42.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.730829 7446 sys_catalog.cc:263] Verifying existing consensus state
20:16:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.737815 7446 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: Bootstrap starting.
20:16:42.775 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.781708 7446 log.cc:824] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: Log is configured to *not* fsync() on all Append() calls
20:16:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.787009 7446 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:16:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.788013 7446 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: Bootstrap complete.
20:16:42.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.822189 7446 raft_consensus.cc:348] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:42.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.826088 7446 raft_consensus.cc:374] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:16:42.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.826421 7446 raft_consensus.cc:729] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 79153dd47fc74902ae24f34e1a774830, State: Initialized, Role: FOLLOWER
20:16:42.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.827527 7446 consensus_queue.cc:260] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:42.831 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:42.839 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.199.188:44655(127.6.199.188:44655)] connecting to peer
20:16:42.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.839875 7451 sys_catalog.cc:455] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } } }
20:16:42.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.840814 7451 sys_catalog.cc:458] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [sys.catalog]: This master's current role is: FOLLOWER
20:16:42.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.841859 7446 sys_catalog.cc:564] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [sys.catalog]: configured and running, proceeding with master startup.
20:16:42.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:42.897172 7400 tablet.cc:2369] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:16:42.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:42.907761 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:42.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:42.908124 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:42.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:42.920 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:42.922 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:42.923 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:42.923 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:42.925 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.199.188:44655(127.6.199.188:44655)] Successfully connected to peer
20:16:42.927 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:42.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.910104 7376 master_runner.cc:186] Error getting master registration for 127.6.199.190:39305: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111),
20:16:42.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:42.942996 7376 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:16:43.024 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:16:43.036 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.6.199.188
20:16:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:43.057016 7451 raft_consensus.cc:482] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:16:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:43.057610 7451 raft_consensus.cc:504] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:43.061679 7451 leader_election.cc:288] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305), 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517)
20:16:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:43.065091 7398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111)
20:16:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:43.075865 7398 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305): Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111)
20:16:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:43.077939 7395 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517): Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:43.078377 7395 leader_election.cc:302] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 79153dd47fc74902ae24f34e1a774830; no voters: 3098cb31b4e3430b8994ba9509a1ce7b, d29e5163e05447b7a40f3396092bec67
20:16:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:43.079692 7451 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:16:43.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:43 dist-test-slave-7k6f krb5kdc[6955](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508197, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.199.188@KRBTEST.COM
20:16:43.293 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1b85c7b1, L:/127.0.0.1:51246 - R:/127.6.199.188:44655] using SASL/GSSAPI
20:16:43.309 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.326 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.328 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.333 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.333 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.334 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.340 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.346 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.366 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.371 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.371 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.371 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.372 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.375 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.375 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.377 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.386 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.388 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.388 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.390 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.395 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.405 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.406 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.446 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.450 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.454 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.526 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.527 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.530 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.530 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.536 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.665 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.671 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.676 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.846 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.847 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.851 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.851 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.853 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.853 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.854 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.854 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:43.911350 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:43.911659 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:43.925 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:43.926 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:43.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.928 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:43.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:43.934 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:44.606 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:44.607 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:44.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:44.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:44.608 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:44.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:44.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:44.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:44.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:44.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:44.817920 7471 raft_consensus.cc:482] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:16:44.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:44.819516 7471 raft_consensus.cc:504] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:44.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:44.823617 7471 leader_election.cc:288] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305), 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517)
20:16:44.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:44.828960 7398 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305): Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111)
20:16:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:44.829748 7395 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517): Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:44.830071 7395 leader_election.cc:302] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 79153dd47fc74902ae24f34e1a774830; no voters: 3098cb31b4e3430b8994ba9509a1ce7b, d29e5163e05447b7a40f3396092bec67
20:16:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:44.830601 7471 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:16:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:44.914500 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:44.914777 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:45.445 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:45.446 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:45.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:45.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:45.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:45.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:45.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:45.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:45.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:45.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:45.800139 6294 debug-util.cc:398] Leaking SignalData structure 0x7b08000a2da0 after lost signal to thread 6231
20:16:45.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:45.800709 6294 debug-util.cc:398] Leaking SignalData structure 0x7b08000c4360 after lost signal to thread 6297
20:16:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:45.918975 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:45.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:45.919322 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:46.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:46.923502 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:46.923833 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:47.534302 7477 raft_consensus.cc:482] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:16:47.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:47.534725 7477 raft_consensus.cc:504] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:47.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:47.536069 7477 leader_election.cc:288] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305), 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517)
20:16:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:47.541034 7398 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305): Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111)
20:16:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:47.541496 7395 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517): Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:47.541824 7395 leader_election.cc:302] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 79153dd47fc74902ae24f34e1a774830; no voters: 3098cb31b4e3430b8994ba9509a1ce7b, d29e5163e05447b7a40f3396092bec67
20:16:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:47.542399 7477 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:16:47.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:47.926549 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:47.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:47.926856 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:47.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:47.946244 7376 master_runner.cc:186] Error getting master registration for 127.6.199.190:39305: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111),
20:16:47.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:47.946702 7376 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:16:48.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:48.929574 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:48.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:48.929878 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:49.345 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:49.346 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:49.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:49.348 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:49.348 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:49.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:49.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:49.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:49.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:49.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:49.831907 6787 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0080 after lost signal to thread 6661
20:16:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:49.832782 6787 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0aa0 after lost signal to thread 6790
20:16:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:49.932842 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:49.933203 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:50.026 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:50.027 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:50.028 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:50.028 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:50.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:50.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:50.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:50.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:50.033 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:50.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:50.937415 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:50.937778 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:51.941314 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:51.941686 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:51.974186 7487 raft_consensus.cc:482] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:16:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:51.974773 7487 raft_consensus.cc:504] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:51.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:51.976505 7487 leader_election.cc:288] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305), 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517)
20:16:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:51.979485 7398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:16:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:51.982316 7395 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517): Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:51.982841 7398 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305): Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111)
20:16:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:51.983212 7398 leader_election.cc:302] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 79153dd47fc74902ae24f34e1a774830; no voters: 3098cb31b4e3430b8994ba9509a1ce7b, d29e5163e05447b7a40f3396092bec67
20:16:51.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:51.983896 7487 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:16:52.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:16:52 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508212, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:16:52.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:52.553751 7390 init.cc:295] Successfully reacquired a new kerberos TGT
20:16:52.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:52.944751 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:52.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:52.945125 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:52.949649 7376 master_runner.cc:186] Error getting master registration for 127.6.199.190:39305: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111),
20:16:52.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:52.950068 7376 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:16:53.445 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:53.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:53.447 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:53.447 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:53.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:53.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:53.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:53.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:53.454 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:53.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:53.947965 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:53.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:53.948237 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:54.426 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:54.427 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:54.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:54.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:54.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:54.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:54.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:54.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:54.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:54.845 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:54.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:54.848 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:54.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:54.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:54.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:54.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:54.851 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:54.854 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:54.950935 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:54.951249 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:55.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:55.747267 7493 raft_consensus.cc:482] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:16:55.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:55.747668 7493 raft_consensus.cc:504] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:16:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:55.749254 7493 leader_election.cc:288] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305), 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517)
20:16:55.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:55.753659 7398 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305): Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111)
20:16:55.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:55.754076 7395 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517): Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:16:55.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:55.754455 7395 leader_election.cc:302] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 79153dd47fc74902ae24f34e1a774830; no voters: 3098cb31b4e3430b8994ba9509a1ce7b, d29e5163e05447b7a40f3396092bec67
20:16:55.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:55.755179 7493 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:16:55.946 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:16:55.947 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:16:55.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:55.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:55.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:16:55.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:55.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:55.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:16:55.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:16:55.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:55.954191 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:55.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:55.954527 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:56.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:56.957398 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:56.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:56.957726 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:57.960563 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:57.960914 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:57.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:57.985728 7376 master_runner.cc:186] Error getting master registration for 127.6.199.190:39305: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111),
20:16:57.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:16:57.986124 7376 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:16:58.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:58.168900 7442 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1580 after lost signal to thread 7378
20:16:58.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:58.169433 7442 debug-util.cc:398] Leaking SignalData structure 0x7b08000aee40 after lost signal to thread 7445
20:16:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:58.965221 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:58.965896 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:16:59.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:59.969127 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:16:59.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:16:59.969395 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:00.005 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:17:00.007 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:17:00.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:00.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:00.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:00.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:00.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:00.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:00.012 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:17:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:00.972196 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:00.972478 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:01.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:01.976176 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:01.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:01.976528 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:02.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 20:17:02 dist-test-slave-7k6f krb5kdc[6955](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714508222, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.199.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:02.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:02.556381 7390 init.cc:295] Successfully reacquired a new kerberos TGT
20:17:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:02.979552 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:02.979835 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:02.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:02.989547 7376 master_runner.cc:186] Error getting master registration for 127.6.199.190:39305: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111),
20:17:02.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:02.989982 7376 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:17:03.726 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:17:03.727 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:17:03.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:03.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:03.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:03.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:03.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:03.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:03.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:17:03.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:17:03.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:17:03.747 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:03.747 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:03.747 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:03.748 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:03.748 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:03.748 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:03.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:17:03.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:03.982901 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:03.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:03.983183 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:04.366 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:17:04.367 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:17:04.367 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:04.367 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:04.368 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:04.368 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:04.368 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:04.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:04.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:17:04.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:04.886967 7508 raft_consensus.cc:482] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:17:04.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:04.887526 7508 raft_consensus.cc:504] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:17:04.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:04.889235 7508 leader_election.cc:288] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305), 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517)
20:17:04.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:04.891516 7398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:17:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:04.894160 7395 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517): Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:17:04.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:04.894560 7398 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305): Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111)
20:17:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:04.894877 7398 leader_election.cc:302] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 79153dd47fc74902ae24f34e1a774830; no voters: 3098cb31b4e3430b8994ba9509a1ce7b, d29e5163e05447b7a40f3396092bec67
20:17:04.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:04.895535 7508 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:17:04.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:04.986130 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:04.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:04.986397 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:05.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:05.418373 6448 debug-util.cc:398] Leaking SignalData structure 0x7b08000c1f20 after lost signal to thread 6380
20:17:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:05.419078 6448 debug-util.cc:398] Leaking SignalData structure 0x7b080009d580 after lost signal to thread 6451
20:17:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:05.990545 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:05.990984 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:06.165 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:17:06.166 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:17:06.166 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:06.167 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:06.167 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:06.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:06.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:06.168 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:06.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:17:06.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:06.995481 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:06.995843 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:07.562125 7515 raft_consensus.cc:482] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:17:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:07.562568 7515 raft_consensus.cc:504] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d29e5163e05447b7a40f3396092bec67" member_type: VOTER last_known_addr { host: "127.6.199.190" port: 39305 } } peers { permanent_uuid: "3098cb31b4e3430b8994ba9509a1ce7b" member_type: VOTER last_known_addr { host: "127.6.199.189" port: 42517 } } peers { permanent_uuid: "79153dd47fc74902ae24f34e1a774830" member_type: VOTER last_known_addr { host: "127.6.199.188" port: 44655 } }
20:17:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:07.564155 7515 leader_election.cc:288] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305), 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517)
20:17:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:07.568985 7398 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d29e5163e05447b7a40f3396092bec67 (127.6.199.190:39305): Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111)
20:17:07.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:07.569458 7395 leader_election.cc:334] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3098cb31b4e3430b8994ba9509a1ce7b (127.6.199.189:42517): Network error: Client connection negotiation failed: client connection to 127.6.199.189:42517: connect: Connection refused (error 111)
20:17:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:07.569748 7395 leader_election.cc:302] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 79153dd47fc74902ae24f34e1a774830; no voters: 3098cb31b4e3430b8994ba9509a1ce7b, d29e5163e05447b7a40f3396092bec67
20:17:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:07.570263 7515 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:17:07.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:07.992453 7376 master_runner.cc:186] Error getting master registration for 127.6.199.190:39305: Network error: Client connection negotiation failed: client connection to 127.6.199.190:39305: connect: Connection refused (error 111),
20:17:07.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:07.992813 7376 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:17:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:07.998544 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:07.998880 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:08.826 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:17:08.827 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:17:08.827 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:08.827 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:08.827 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:08.828 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:08.828 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:08.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:08.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:17:09.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:09.001565 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:09.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:09.001832 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:10.004686 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:10.004971 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:11.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:11.007704 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:11.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:11.007975 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:12.011377 7463 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 79153dd47fc74902ae24f34e1a774830: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 20:17:12.011714 7463 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:12.428 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.189:42517(127.6.199.189:42517)] connecting to peer
20:17:12.429 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.199.190:39305(127.6.199.190:39305)] connecting to peer
20:17:12.430 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:12.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.199.189:42517(127.6.199.189:42517)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:12.430 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.189:42517: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517
20:17:12.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:12.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.190:39305(127.6.199.190:39305)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:12.431 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.199.190:39305: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
20:17:12.434 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655; will retry
20:17:12.439 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29711), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [190ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [602ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [609ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [613ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [616ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [640ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [644ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [646ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [658ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [662ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [663ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [677ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [679ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [682ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [721ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [723ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [799ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [801ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [804ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [934ms] refreshing cache from master, [934ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [939ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [942ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [944ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1114ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1123ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1194ms] refreshing cache from master, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1874ms] refreshing cache from master, [1874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1877ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1880ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1882ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2714ms] refreshing cache from master, [2714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [2715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [2717ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [2718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2720ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [2721ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [6614ms] refreshing cache from master, [6614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [6615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [6616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [6617ms] too many traces: truncated at 100 traces, deferred=Deferred@923163143(state=PENDING, result=null, callback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769), errback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769)))
20:17:12.445 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.199.190:39305(127.6.199.190:39305)] disconnecting while in state TERMINATED
20:17:12.447 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.199.189:42517(127.6.199.189:42517)] disconnecting while in state TERMINATED
20:17:12.447 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.199.188:44655(127.6.199.188:44655)] disconnecting while in state READY
20:17:12.456 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:17:12.472 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.199.188:44655(127.6.199.188:44655)] handling channelInactive
20:17:12.473 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.199.188:44655(127.6.199.188:44655)] cleaning up while in state READY due to: connection closed
20:17:12.473 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:17:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:12.490085 6942 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 7376
20:17:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T20:17:13Z chronyd exiting
20:17:14.613 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:17:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:14.614708 6218 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 6499
20:17:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:14.643067 6218 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 6658
20:17:14.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:14.670188 6218 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 6801
20:17:14.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:14.695845 6218 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 6230
20:17:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:14.734100 6218 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 6300
20:17:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 20:17:14.764912 6218 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskASVjAM/build/tsan/bin/kudu with pid 6375
20:17:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T20:17:15Z chronyd exiting
20:17:16.860 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29711), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [190ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [602ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [609ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [613ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [616ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [640ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [644ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [646ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [658ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [662ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [663ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [677ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [679ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [682ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [721ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [723ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [799ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [801ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [804ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [934ms] refreshing cache from master, [934ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [939ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [942ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [944ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1114ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1123ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1194ms] refreshing cache from master, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1874ms] refreshing cache from master, [1874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1877ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1880ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1882ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2714ms] refreshing cache from master, [2714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [2715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [2717ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [2718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2720ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [2721ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [6614ms] refreshing cache from master, [6614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [6615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [6616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [6617ms] too many traces: truncated at 100 traces, deferred=Deferred@923163143(state=PENDING, result=null, callback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769), errback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.199.190:39305
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29711), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [190ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [602ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [609ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [613ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [616ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [640ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [644ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [646ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [658ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [662ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [663ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [677ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [679ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [682ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [721ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [723ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [799ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [801ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [804ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [934ms] refreshing cache from master, [934ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [939ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [942ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [944ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1114ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1123ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1194ms] refreshing cache from master, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1874ms] refreshing cache from master, [1874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1877ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1880ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1882ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2714ms] refreshing cache from master, [2714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [2715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [2717ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [2718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2720ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [2721ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [6614ms] refreshing cache from master, [6614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [6615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [6616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [6617ms] too many traces: truncated at 100 traces, deferred=Deferred@923163143(state=PENDING, result=null, callback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769), errback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.199.190:39305
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
20:17:17.400 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:17:17.400 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29711), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [190ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [602ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [609ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [613ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [616ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [640ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [644ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [646ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [658ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [662ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [663ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [677ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [679ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [682ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [721ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [723ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [799ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [801ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [804ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [934ms] refreshing cache from master, [934ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [939ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [942ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [944ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1114ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1123ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1194ms] refreshing cache from master, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1874ms] refreshing cache from master, [1874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1877ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1880ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1882ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2714ms] refreshing cache from master, [2714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [2715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [2717ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [2718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2720ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [2721ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [6614ms] refreshing cache from master, [6614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [6615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [6616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [6617ms] too many traces: truncated at 100 traces, deferred=Deferred@923163143(state=PENDING, result=null, callback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769), errback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.199.190:39305
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29711), Traces: [0ms] refreshing cache from master, [32ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [190ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [602ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [609ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [613ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [616ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [640ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [644ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [646ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [658ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [662ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [663ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [677ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [679ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [682ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [721ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [723ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [799ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [801ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [804ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [934ms] refreshing cache from master, [934ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [939ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [942ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [944ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [945ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1114ms] refreshing cache from master, [1114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1123ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1194ms] refreshing cache from master, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1197ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1204ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1874ms] refreshing cache from master, [1874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [1877ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [1880ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [1882ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2714ms] refreshing cache from master, [2714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [2715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [2716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [2717ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.189:42517: Network error: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517, [2718ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.190:39305: Network error: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [2720ms] Sub RPC ConnectToMaster: received response from server master-127.6.199.188:44655: OK, [2721ms] delaying RPC due to: Service unavailable: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305, [6614ms] refreshing cache from master, [6614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.189:42517, [6615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.190:39305, [6616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.199.188:44655, [6617ms] too many traces: truncated at 100 traces, deferred=Deferred@923163143(state=PENDING, result=null, callback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769), errback=(continuation of Deferred@1257829443 after retry RPC after error@1073620767) -> (continuation of Deferred@611778940 after retry RPC after error@564437399) -> (continuation of Deferred@745734924 after retry RPC after error@1652236098) -> (continuation of Deferred@2115739280 after retry RPC after error@2050437341) -> (continuation of Deferred@1323841524 after retry RPC after error@1951637604) -> (continuation of Deferred@1453215584 after retry RPC after error@367803574) -> (continuation of Deferred@439680616 after retry RPC after error@1930247626) -> (continuation of Deferred@1473022340 after retry RPC after error@598808135) -> (continuation of Deferred@508263399 after retry RPC after error@70444681) -> (continuation of Deferred@922865442 after retry RPC after error@472511659) -> (continuation of Deferred@1501048970 after retry RPC after error@1257083778) -> (continuation of Deferred@1932274978 after retry RPC after error@544408384) -> (continuation of Deferred@941067172 after retry RPC after error@515931815) -> (continuation of Deferred@211228000 after retry RPC after error@885419697) -> (continuation of Deferred@805869862 after retry RPC after error@765521094) -> (continuation of Deferred@430493703 after retry RPC after error@513578597) -> (continuation of Deferred@1781136697 after retry RPC after error@1078323814) -> (continuation of Deferred@531772033 after retry RPC after error@1701290156) -> (continuation of Deferred@1082280204 after retry RPC after error@796630432) -> (continuation of Deferred@567510260 after retry RPC after error@2036937378) -> (continuation of Deferred@728882858 after retry RPC after error@1477755129) -> (continuation of Deferred@1708989227 after retry RPC after error@1427228944) -> (continuation of Deferred@1890634134 after retry RPC after error@241032305) -> (continuation of Deferred@64260818 after retry RPC after error@1482070769)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.199.189:42517,127.6.199.190:39305,127.6.199.188:44655) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.189:42517(127.6.199.189:42517): Connection refused: /127.6.199.189:42517,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.199.190:39305(127.6.199.190:39305): Connection refused: /127.6.199.190:39305
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.199.190:39305
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)