06:59:02.512 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:59:02.529 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskhc0eXq/test-tmp
06:59:02.532 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:59:02.537 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskhc0eXq/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:59:02.767 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
06:59:02.896 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:59:02.898 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:59:02.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:02.901715 15055 mini_oidc.cc:152] Starting JWKS server
06:59:02.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:02.905462 15055 webserver.cc:460] Webserver started at http://127.0.0.1:39947/ using document root <none> and password file <none>
06:59:02.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:02.905977 15055 mini_oidc.cc:189] Starting OIDC Discovery server
06:59:02.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:02.908463 15055 webserver.cc:460] Webserver started at http://127.0.0.1:43001/ using document root <none> and password file <none>
06:59:02.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:02Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:59:02.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:02Z Disabled control of system clock
06:59:02.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:02.972373 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:02.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:02.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/wal
06:59:02.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/data
06:59:02.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:02.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:02.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/logs
06:59:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/data/info.pb
06:59:02.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:02.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:02.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:02.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:02.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:02.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:02.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:02.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:02.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:02.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:02.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.254:41005
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.254
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json with env {}
06:59:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.269621 15067 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.270213 15067 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.270939 15067 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.300087 15067 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.300362 15067 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.300550 15067 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:03.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.300781 15067 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:03.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.301088 15067 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:03.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.301321 15067 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39947/jwks.json
06:59:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:03.335842 15067 master_runner.cc:386] Master server non-default flags:
06:59:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/data
06:59:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/wal
06:59:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.254:41005
06:59:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json
06:59:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/data/info.pb
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.254
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/logs
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:03.337468 15067 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:03.339684 15067 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.356653 15073 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:04.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.357111 15074 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:04.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:03.358738 15067 server_base.cc:1034] running on GCE node
06:59:04.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:03.357832 15076 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:04.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.452960 15067 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:04.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.455580 15067 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:04.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.456946 15067 hybrid_clock.cc:648] HybridClock initialized: now 1714114744456897 us; error 57 us; skew 500 ppm
06:59:04.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.457819 15067 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.472055 15067 webserver.cc:460] Webserver started at http://127.14.179.254:44527/ using document root <none> and password file <none>
06:59:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.473151 15067 fs_manager.cc:359] Metadata directory not provided
06:59:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.473379 15067 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:04.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.473938 15067 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.478940 15067 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/data/instance:
06:59:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c71294f9f7408f8d5522085a1d53f6"
06:59:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:04 on dist-test-slave-4c6b"
06:59:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.480319 15067 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/wal/instance:
06:59:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c71294f9f7408f8d5522085a1d53f6"
06:59:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:04 on dist-test-slave-4c6b"
06:59:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.487715 15067 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
06:59:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.493321 15083 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.494339 15067 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
06:59:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.494704 15067 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/wal
06:59:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18c71294f9f7408f8d5522085a1d53f6"
06:59:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:04 on dist-test-slave-4c6b"
06:59:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.495114 15067 fs_report.cc:383] FS layout report
06:59:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/wal
06:59:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/wal
06:59:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/data/data
06:59:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:04.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.558240 15067 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.559918 15067 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:04.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.560446 15067 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:04.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.634732 15067 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.254:41005
06:59:04.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.634828 15134 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.254:41005 every 8 connection(s)
06:59:04.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.637542 15067 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/data/info.pb
06:59:04.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.642725 15135 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:04.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.643918 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15067
06:59:04.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.644366 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-0/wal/instance
06:59:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.647779 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/wal
06:59:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/data
06:59:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/logs
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/data/info.pb
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.253:37719
06:59:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.253
06:59:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:04.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:04.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json with env {}
06:59:04.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.655371 15135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } has no permanent_uuid. Determining permanent_uuid...
06:59:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.684559 15135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } has no permanent_uuid. Determining permanent_uuid...
06:59:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.686803 15085 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.179.253:37719: connect: Connection refused (error 111)
06:59:04.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.690932 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:37719: Network error: Client connection negotiation failed: client connection to 127.14.179.253:37719: connect: Connection refused (error 111)
06:59:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.740579 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } attempt: 1
06:59:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.744724 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:37719: Network error: Client connection negotiation failed: client connection to 127.14.179.253:37719: connect: Connection refused (error 111)
06:59:04.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.813299 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } attempt: 2
06:59:04.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.817405 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:37719: Network error: Client connection negotiation failed: client connection to 127.14.179.253:37719: connect: Connection refused (error 111)
06:59:04.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:04.908871 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } attempt: 3
06:59:04.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.912827 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:37719: Network error: Client connection negotiation failed: client connection to 127.14.179.253:37719: connect: Connection refused (error 111)
06:59:04.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.961866 15137 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:04.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.962476 15137 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:04.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.962986 15137 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.992136 15137 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.992436 15137 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.992643 15137 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.992826 15137 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:04.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.993021 15137 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:04.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:04.993165 15137 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39947/jwks.json
06:59:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:05.026038 15137 master_runner.cc:386] Master server non-default flags:
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/data
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/wal
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:05.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.253:37719
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/data/info.pb
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.253
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/logs
06:59:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:05.027386 15137 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:05.029011 15137 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:05.044775 15146 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:05.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:05.056305 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } attempt: 4
06:59:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:05.060703 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:37719: Network error: Client connection negotiation failed: client connection to 127.14.179.253:37719: connect: Connection refused (error 111)
06:59:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:05.360170 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } attempt: 5
06:59:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:05.365628 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:37719: Network error: Client connection negotiation failed: client connection to 127.14.179.253:37719: connect: Connection refused (error 111)
06:59:05.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:05.913064 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } attempt: 6
06:59:05.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:05.924072 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:37719: Network error: Client connection negotiation failed: client connection to 127.14.179.253:37719: connect: Connection refused (error 111)
06:59:06.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:05.045217 15149 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:06.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:05.046460 15137 server_base.cc:1034] running on GCE node
06:59:06.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:05.045408 15147 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:06.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.129738 15137 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:06.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.133132 15137 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:06.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.134583 15137 hybrid_clock.cc:648] HybridClock initialized: now 1714114746134537 us; error 61 us; skew 500 ppm
06:59:06.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.135634 15137 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.141933 15137 webserver.cc:460] Webserver started at http://127.14.179.253:35291/ using document root <none> and password file <none>
06:59:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.142865 15137 fs_manager.cc:359] Metadata directory not provided
06:59:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.143065 15137 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.143487 15137 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:06.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.148283 15137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/data/instance:
06:59:06.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0139e554dec45d0b5b5294615f82fd3"
06:59:06.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:06 on dist-test-slave-4c6b"
06:59:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.149359 15137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/wal/instance:
06:59:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0139e554dec45d0b5b5294615f82fd3"
06:59:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:06 on dist-test-slave-4c6b"
06:59:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.156518 15137 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
06:59:06.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.162014 15158 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:06.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.163179 15137 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:59:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.163697 15137 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/wal
06:59:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0139e554dec45d0b5b5294615f82fd3"
06:59:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:06 on dist-test-slave-4c6b"
06:59:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.164060 15137 fs_report.cc:383] FS layout report
06:59:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/wal
06:59:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/wal
06:59:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/data/data
06:59:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:06.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:06.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.228152 15137 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.229563 15137 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:06.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.229993 15137 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:06.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.297694 15137 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.253:37719
06:59:06.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.297792 15209 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.253:37719 every 8 connection(s)
06:59:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.300513 15137 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/data/info.pb
06:59:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.304963 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15137
06:59:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.305328 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-1/wal/instance
06:59:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.305805 15210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.307734 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/wal
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/data
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/logs
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/data/info.pb
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.252:42273
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.252
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json with env {}
06:59:06.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.323400 15210 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } has no permanent_uuid. Determining permanent_uuid...
06:59:06.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.349517 15210 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } has no permanent_uuid. Determining permanent_uuid...
06:59:06.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.363830 15210 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } has no permanent_uuid. Determining permanent_uuid...
06:59:06.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.366724 15162 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.370898 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.420370 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 1
06:59:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.424398 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:06.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.492758 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 2
06:59:06.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.497359 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:06.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.588812 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 3
06:59:06.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.592619 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:06.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.620842 15212 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:06.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.621412 15212 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:06.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.622043 15212 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.650908 15212 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.651162 15212 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.651337 15212 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.651508 15212 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.651757 15212 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:06.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.651934 15212 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39947/jwks.json
06:59:06.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.684223 15212 master_runner.cc:386] Master server non-default flags:
06:59:06.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:06.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/data
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/wal
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.252:42273
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/data/info.pb
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.252
06:59:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/logs
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:06.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:06.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.685479 15212 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:06.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.687345 15212 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:06.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.704205 15222 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.736016 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 4
06:59:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.740660 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:06.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.984606 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } attempt: 7
06:59:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.009661 15135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } has no permanent_uuid. Determining permanent_uuid...
06:59:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:07.014791 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:07.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.040071 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 5
06:59:07.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:07.044605 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:07.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.073547 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 1
06:59:07.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:07.077911 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.159613 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 2
06:59:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:07.163830 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:07.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.249253 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 3
06:59:07.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:07.254081 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:07.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.394481 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 4
06:59:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:07.399358 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:07.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.592002 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 6
06:59:07.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:07.598260 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.682704 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 5
06:59:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:07.686902 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:42273: Network error: Client connection negotiation failed: client connection to 127.14.179.252:42273: connect: Connection refused (error 111)
06:59:07.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.705021 15225 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:07.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:06.706797 15212 server_base.cc:1034] running on GCE node
06:59:07.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:06.704660 15223 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.798957 15212 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.802047 15212 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:07.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.803469 15212 hybrid_clock.cc:648] HybridClock initialized: now 1714114747803443 us; error 45 us; skew 500 ppm
06:59:07.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.804225 15212 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:07.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.810719 15212 webserver.cc:460] Webserver started at http://127.14.179.252:46839/ using document root <none> and password file <none>
06:59:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.811571 15212 fs_manager.cc:359] Metadata directory not provided
06:59:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.811779 15212 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:07.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.812160 15212 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.816880 15212 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/data/instance:
06:59:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1162f712f5e49cdad5c992ec5523fcf"
06:59:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:07 on dist-test-slave-4c6b"
06:59:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.817884 15212 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/wal/instance:
06:59:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1162f712f5e49cdad5c992ec5523fcf"
06:59:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:07 on dist-test-slave-4c6b"
06:59:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.825014 15212 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
06:59:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.830427 15237 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.831521 15212 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
06:59:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.831861 15212 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/wal
06:59:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a1162f712f5e49cdad5c992ec5523fcf"
06:59:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:07 on dist-test-slave-4c6b"
06:59:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.832274 15212 fs_report.cc:383] FS layout report
06:59:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/wal
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/wal
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/data/data
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.881623 15212 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.883234 15212 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.883735 15212 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.952286 15212 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.252:42273
06:59:07.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.952366 15288 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.252:42273 every 8 connection(s)
06:59:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.955132 15212 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/data/info.pb
06:59:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.959623 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15212
06:59:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.960165 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/master-2/wal/instance
06:59:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.960891 15289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:07.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:07.979209 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } has no permanent_uuid. Determining permanent_uuid...
06:59:08.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.003908 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } has no permanent_uuid. Determining permanent_uuid...
06:59:08.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.013190 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } has no permanent_uuid. Determining permanent_uuid...
06:59:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.027478 15289 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf: Bootstrap starting.
06:59:08.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.033794 15289 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf: Neither blocks nor log segments found. Creating new log.
06:59:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.035669 15289 log.cc:824] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf: Log is configured to *not* fsync() on all Append() calls
06:59:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.040539 15289 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf: No bootstrap required, opened a new log
06:59:08.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.062181 15289 raft_consensus.cc:348] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.062858 15289 raft_consensus.cc:374] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.063062 15289 raft_consensus.cc:729] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a1162f712f5e49cdad5c992ec5523fcf, State: Initialized, Role: FOLLOWER
06:59:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.063742 15289 consensus_queue.cc:260] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.066119 15298 sys_catalog.cc:455] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.066875 15298 sys_catalog.cc:458] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: This master's current role is: FOLLOWER
06:59:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.067618 15289 sys_catalog.cc:564] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: configured and running, proceeding with master startup.
06:59:08.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:08.091784 15309 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:08.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:08.092161 15309 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:08.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.089859 15212 master_runner.cc:186] Error getting master registration for 127.14.179.254:41005: OK, instance_id { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" instance_seqno: 1714114744615302 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:59:08.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.119295 15212 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:08.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:08.138893 15243 tablet.cc:2369] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.239272 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 6
06:59:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.258255 15135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: Bootstrap starting.
06:59:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.266225 15135 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: Neither blocks nor log segments found. Creating new log.
06:59:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.268648 15135 log.cc:824] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: Log is configured to *not* fsync() on all Append() calls
06:59:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.273494 15135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: No bootstrap required, opened a new log
06:59:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.298786 15298 raft_consensus.cc:482] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.299382 15298 raft_consensus.cc:504] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.301976 15298 leader_election.cc:288] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 18c71294f9f7408f8d5522085a1d53f6 (127.14.179.254:41005), e0139e554dec45d0b5b5294615f82fd3 (127.14.179.253:37719)
06:59:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.306217 15135 raft_consensus.cc:348] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.307135 15135 raft_consensus.cc:374] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:08.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.307418 15135 raft_consensus.cc:729] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18c71294f9f7408f8d5522085a1d53f6, State: Initialized, Role: FOLLOWER
06:59:08.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.308290 15135 consensus_queue.cc:260] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.316258 15135 sys_catalog.cc:564] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [sys.catalog]: configured and running, proceeding with master startup.
06:59:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.316455 15110 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a1162f712f5e49cdad5c992ec5523fcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "18c71294f9f7408f8d5522085a1d53f6" is_pre_election: true
06:59:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.317389 15110 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a1162f712f5e49cdad5c992ec5523fcf in term 0.
06:59:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.318879 15239 leader_election.cc:302] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 18c71294f9f7408f8d5522085a1d53f6, a1162f712f5e49cdad5c992ec5523fcf; no voters:
06:59:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:08.319558 15089 tablet.cc:2369] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.319801 15298 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 0 FOLLOWER]: Leader pre-election won for term 1
06:59:08.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.320176 15298 raft_consensus.cc:482] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:59:08.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.320482 15298 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 0 FOLLOWER]: Advancing to term 1
06:59:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.315068 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.324170 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [sys.catalog]: This master's current role is: FOLLOWER
06:59:08.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.335402 15298 raft_consensus.cc:504] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.340448 15298 leader_election.cc:288] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [CANDIDATE]: Term 1 election: Requested vote from peers 18c71294f9f7408f8d5522085a1d53f6 (127.14.179.254:41005), e0139e554dec45d0b5b5294615f82fd3 (127.14.179.253:37719)
06:59:08.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.342830 15110 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a1162f712f5e49cdad5c992ec5523fcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "18c71294f9f7408f8d5522085a1d53f6"
06:59:08.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.343477 15110 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [term 0 FOLLOWER]: Advancing to term 1
06:59:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.351753 15110 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a1162f712f5e49cdad5c992ec5523fcf in term 1.
06:59:08.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.352939 15239 leader_election.cc:302] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 18c71294f9f7408f8d5522085a1d53f6, a1162f712f5e49cdad5c992ec5523fcf; no voters:
06:59:08.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.354102 15298 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 1 FOLLOWER]: Leader election won for term 1
06:59:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.356323 15298 raft_consensus.cc:686] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [term 1 LEADER]: Becoming Leader. State: Replica: a1162f712f5e49cdad5c992ec5523fcf, State: Running, Role: LEADER
06:59:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.357313 15298 consensus_queue.cc:237] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.378700 15320 sys_catalog.cc:455] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: SysCatalogTable state changed. Reason: New leader a1162f712f5e49cdad5c992ec5523fcf. Latest consensus state: current_term: 1 leader_uuid: "a1162f712f5e49cdad5c992ec5523fcf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.384518 15320 sys_catalog.cc:458] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: This master's current role is: LEADER
06:59:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.388300 15327 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:59:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.395689 15327 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:59:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:08.420678 15326 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:08.421057 15326 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.443008 15110 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [term 1 FOLLOWER]: Refusing update from remote peer a1162f712f5e49cdad5c992ec5523fcf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:59:08.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.445531 15320 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:59:08.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.392552 15067 master_runner.cc:186] Error getting master registration for 127.14.179.253:37719: OK, instance_id { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" instance_seqno: 1714114746279595 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:59:08.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.450456 15067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.485661 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a1162f712f5e49cdad5c992ec5523fcf. Latest consensus state: current_term: 1 leader_uuid: "a1162f712f5e49cdad5c992ec5523fcf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.486500 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [sys.catalog]: This master's current role is: FOLLOWER
06:59:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.506232 15327 catalog_manager.cc:1347] Generated new cluster ID: 0cf069c3ab684eb4b62dc2785c233f13
06:59:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.506613 15327 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:59:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.509685 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a1162f712f5e49cdad5c992ec5523fcf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.510461 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6 [sys.catalog]: This master's current role is: FOLLOWER
06:59:08.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.521661 15298 sys_catalog.cc:455] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a1162f712f5e49cdad5c992ec5523fcf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.522430 15298 sys_catalog.cc:458] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: This master's current role is: LEADER
06:59:08.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.563717 15327 catalog_manager.cc:1370] Generated new certificate authority record
06:59:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.568439 15327 catalog_manager.cc:1504] Loading token signing keys...
06:59:08.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.608242 15327 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf: Generated new TSK 0
06:59:08.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.609355 15327 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:59:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.634670 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/wal
06:59:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/data
06:59:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/logs
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/data/info.pb
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.193:0
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.193
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.193
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:59:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.658701 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } attempt: 7
06:59:08.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.710481 15210 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3: Bootstrap starting.
06:59:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.726706 15210 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3: Neither blocks nor log segments found. Creating new log.
06:59:08.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.732283 15210 log.cc:824] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3: Log is configured to *not* fsync() on all Append() calls
06:59:08.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.754068 15210 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3: No bootstrap required, opened a new log
06:59:08.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.830407 15210 raft_consensus.cc:348] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.833321 15210 raft_consensus.cc:374] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:08.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.838419 15210 raft_consensus.cc:729] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0139e554dec45d0b5b5294615f82fd3, State: Initialized, Role: FOLLOWER
06:59:08.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.841204 15210 consensus_queue.cc:260] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } }
06:59:08.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.863998 15185 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a1162f712f5e49cdad5c992ec5523fcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0139e554dec45d0b5b5294615f82fd3" is_pre_election: true
06:59:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.864912 15185 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a1162f712f5e49cdad5c992ec5523fcf in term 0.
06:59:08.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.868674 15183 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [term 0 FOLLOWER]: Advancing to term 1
06:59:08.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.867305 15184 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a1162f712f5e49cdad5c992ec5523fcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0139e554dec45d0b5b5294615f82fd3"
06:59:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.878835 15183 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [term 1 FOLLOWER]: Refusing update from remote peer a1162f712f5e49cdad5c992ec5523fcf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:59:08.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.880692 15320 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:59:08.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.887933 15337 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.889042 15210 sys_catalog.cc:564] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [sys.catalog]: configured and running, proceeding with master startup.
06:59:08.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.890266 15337 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [sys.catalog]: This master's current role is: FOLLOWER
06:59:08.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.993932 15338 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a1162f712f5e49cdad5c992ec5523fcf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.995339 15338 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [sys.catalog]: This master's current role is: FOLLOWER
06:59:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.996140 15337 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a1162f712f5e49cdad5c992ec5523fcf. Latest consensus state: current_term: 1 leader_uuid: "a1162f712f5e49cdad5c992ec5523fcf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:08.997555 15337 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3 [sys.catalog]: This master's current role is: FOLLOWER
06:59:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.040475 15354 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
06:59:09.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:09.187623 15334 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:09.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:09.188302 15334 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:09.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:09.189080 15334 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:09.243397 15334 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:09.244850 15334 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.179.193
06:59:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.304320 15334 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/data
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/wal
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.193:0
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/data/info.pb
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.193
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.193
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/logs
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.308257 15334 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:09.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.310637 15334 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.322474 15360 sys_catalog.cc:455] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a1162f712f5e49cdad5c992ec5523fcf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "18c71294f9f7408f8d5522085a1d53f6" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 41005 } } peers { permanent_uuid: "e0139e554dec45d0b5b5294615f82fd3" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 37719 } } peers { permanent_uuid: "a1162f712f5e49cdad5c992ec5523fcf" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 42273 } } }
06:59:09.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.330598 15360 sys_catalog.cc:458] T 00000000000000000000000000000000 P a1162f712f5e49cdad5c992ec5523fcf [sys.catalog]: This master's current role is: LEADER
06:59:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:09.352473 15365 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:09.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.429986 15326 catalog_manager.cc:1259] Loaded cluster ID: 0cf069c3ab684eb4b62dc2785c233f13
06:59:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.431109 15326 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: loading cluster ID for follower catalog manager: success
06:59:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.443897 15326 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: acquiring CA information for follower catalog manager: success
06:59:09.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.454015 15326 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 18c71294f9f7408f8d5522085a1d53f6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:59:10.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.075634 15354 catalog_manager.cc:1259] Loaded cluster ID: 0cf069c3ab684eb4b62dc2785c233f13
06:59:10.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.077773 15354 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3: loading cluster ID for follower catalog manager: success
06:59:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.100837 15354 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3: acquiring CA information for follower catalog manager: success
06:59:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.122515 15354 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e0139e554dec45d0b5b5294615f82fd3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:59:10.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:09.362947 15334 server_base.cc:1034] running on GCE node
06:59:10.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:09.366591 15368 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:09.356837 15363 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.677017 15334 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.687372 15334 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.695704 15334 hybrid_clock.cc:648] HybridClock initialized: now 1714114750695648 us; error 60 us; skew 500 ppm
06:59:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.696806 15334 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.727547 15334 webserver.cc:460] Webserver started at http://127.14.179.193:42699/ using document root <none> and password file <none>
06:59:10.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.729209 15334 fs_manager.cc:359] Metadata directory not provided
06:59:10.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.729729 15334 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:10.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.730461 15334 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:10.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.737641 15334 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/data/instance:
06:59:10.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ddbf1f9a77d44b4c8504c09992241b92"
06:59:10.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:10 on dist-test-slave-4c6b"
06:59:10.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.739439 15334 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/wal/instance:
06:59:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ddbf1f9a77d44b4c8504c09992241b92"
06:59:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:10 on dist-test-slave-4c6b"
06:59:10.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.755254 15334 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.005s sys 0.008s
06:59:10.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.768757 15375 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:10.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.770468 15334 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.006s sys 0.004s
06:59:10.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.771072 15334 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/wal
06:59:10.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ddbf1f9a77d44b4c8504c09992241b92"
06:59:10.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:10 on dist-test-slave-4c6b"
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.771770 15334 fs_report.cc:383] FS layout report
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/wal
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/wal
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/data/data
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:10.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.842523 15334 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:10.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.844535 15334 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:10.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.845082 15334 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:10.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.848423 15334 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.854380 15334 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.854665 15334 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
06:59:10.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.854940 15334 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:10.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:10.855152 15334 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:59:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.095700 15334 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.193:45699
06:59:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.095813 15487 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.193:45699 every 8 connection(s)
06:59:11.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.098918 15334 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/data/info.pb
06:59:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.102594 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15334
06:59:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.103200 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-0/wal/instance
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.133729 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/wal
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/data
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/logs
06:59:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/data/info.pb
06:59:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.194:0
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.194
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.194
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:59:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.237747 15489 heartbeater.cc:347] Connected to a master server at 127.14.179.253:37719
06:59:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.238339 15489 heartbeater.cc:464] Registering TS with master...
06:59:11.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.240031 15489 heartbeater.cc:511] Master 127.14.179.253:37719 requested a full tablet report, sending...
06:59:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.244040 15175 ts_manager.cc:194] Registered new tserver with Master: ddbf1f9a77d44b4c8504c09992241b92 (127.14.179.193:45699)
06:59:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.245046 15488 heartbeater.cc:347] Connected to a master server at 127.14.179.252:42273
06:59:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.245405 15488 heartbeater.cc:464] Registering TS with master...
06:59:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.246305 15488 heartbeater.cc:511] Master 127.14.179.252:42273 requested a full tablet report, sending...
06:59:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.248984 15254 ts_manager.cc:194] Registered new tserver with Master: ddbf1f9a77d44b4c8504c09992241b92 (127.14.179.193:45699)
06:59:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.251863 15254 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.179.193:57107
06:59:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.265810 15493 heartbeater.cc:347] Connected to a master server at 127.14.179.254:41005
06:59:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.266167 15493 heartbeater.cc:464] Registering TS with master...
06:59:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.266884 15493 heartbeater.cc:511] Master 127.14.179.254:41005 requested a full tablet report, sending...
06:59:11.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.270609 15100 ts_manager.cc:194] Registered new tserver with Master: ddbf1f9a77d44b4c8504c09992241b92 (127.14.179.193:45699)
06:59:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:11.567818 15492 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:11.568369 15492 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:11.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:11.568907 15492 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:11.598151 15492 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:11.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:11.599148 15492 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.179.194
06:59:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.632233 15492 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/data
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/wal
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.194:0
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/data/info.pb
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.194
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.194
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/logs
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.633844 15492 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:11.635898 15492 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:11.663552 15505 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:12.257272 15488 heartbeater.cc:503] Master 127.14.179.252:42273 was elected leader, sending a full tablet report...
06:59:13.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:11.664934 15504 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:13.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.056898 15503 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 15492
06:59:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.172508 15492 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.511s user 0.415s sys 0.751s
06:59:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.174571 15492 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.513s user 0.416s sys 0.751s
06:59:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.174912 15507 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:13.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.178849 15506 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1511 milliseconds
06:59:13.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.178884 15492 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:59:13.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.180263 15492 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.182930 15492 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.184356 15492 hybrid_clock.cc:648] HybridClock initialized: now 1714114753184261 us; error 94 us; skew 500 ppm
06:59:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.185395 15492 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.192955 15492 webserver.cc:460] Webserver started at http://127.14.179.194:41975/ using document root <none> and password file <none>
06:59:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.194124 15492 fs_manager.cc:359] Metadata directory not provided
06:59:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.194397 15492 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.194918 15492 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.201766 15492 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/data/instance:
06:59:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4a69de1a99f477eac2395237139ba0d"
06:59:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:13 on dist-test-slave-4c6b"
06:59:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.203161 15492 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/wal/instance:
06:59:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4a69de1a99f477eac2395237139ba0d"
06:59:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:13 on dist-test-slave-4c6b"
06:59:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.212059 15492 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.000s sys 0.008s
06:59:13.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.219319 15515 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.220430 15492 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.220742 15492 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/wal
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4a69de1a99f477eac2395237139ba0d"
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:13 on dist-test-slave-4c6b"
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.221046 15492 fs_report.cc:383] FS layout report
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/wal
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/wal
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/data/data
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:13.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:13.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:13.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:13.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:13.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.280757 15492 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:13.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.282500 15492 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.283018 15492 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.285575 15492 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:13.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.290302 15492 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:13.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.290563 15492 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
06:59:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.290871 15492 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.291102 15492 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:59:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.434077 15492 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.194:39367
06:59:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.434191 15628 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.194:39367 every 8 connection(s)
06:59:13.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.436633 15492 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/data/info.pb
06:59:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.439481 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15492
06:59:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.440102 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-1/wal/instance
06:59:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.461367 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/wal
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/data
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/logs
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/data/info.pb
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.195:0
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.195
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.195
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:59:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.492923 15629 heartbeater.cc:347] Connected to a master server at 127.14.179.252:42273
06:59:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.493412 15629 heartbeater.cc:464] Registering TS with master...
06:59:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.494730 15629 heartbeater.cc:511] Master 127.14.179.252:42273 requested a full tablet report, sending...
06:59:13.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.496510 15631 heartbeater.cc:347] Connected to a master server at 127.14.179.254:41005
06:59:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.496857 15631 heartbeater.cc:464] Registering TS with master...
06:59:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.497654 15254 ts_manager.cc:194] Registered new tserver with Master: d4a69de1a99f477eac2395237139ba0d (127.14.179.194:39367)
06:59:13.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.497784 15631 heartbeater.cc:511] Master 127.14.179.254:41005 requested a full tablet report, sending...
06:59:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.499315 15254 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.179.194:55181
06:59:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.500927 15630 heartbeater.cc:347] Connected to a master server at 127.14.179.253:37719
06:59:13.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.501206 15630 heartbeater.cc:464] Registering TS with master...
06:59:13.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.501010 15100 ts_manager.cc:194] Registered new tserver with Master: d4a69de1a99f477eac2395237139ba0d (127.14.179.194:39367)
06:59:13.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.502516 15630 heartbeater.cc:511] Master 127.14.179.253:37719 requested a full tablet report, sending...
06:59:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.504925 15175 ts_manager.cc:194] Registered new tserver with Master: d4a69de1a99f477eac2395237139ba0d (127.14.179.194:39367)
06:59:13.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.785547 15637 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:13.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.786046 15637 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:13.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.786535 15637 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.817453 15637 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.818290 15637 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.179.195
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.851795 15637 tablet_server_runner.cc:78] Tablet server non-default flags:
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:39479
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/data
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/wal
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.195:0
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/data/info.pb
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.195
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:41005,127.14.179.253:37719,127.14.179.252:42273
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.195
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/logs
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.853384 15637 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:13.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.855119 15637 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.873458 15645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.503589 15629 heartbeater.cc:503] Master 127.14.179.252:42273 was elected leader, sending a full tablet report...
06:59:14.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.873739 15648 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:14.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:13.873770 15646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:14.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:13.874784 15637 server_base.cc:1034] running on GCE node
06:59:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.974510 15637 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.977384 15637 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.978816 15637 hybrid_clock.cc:648] HybridClock initialized: now 1714114754978743 us; error 85 us; skew 500 ppm
06:59:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.979645 15637 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.985575 15637 webserver.cc:460] Webserver started at http://127.14.179.195:41241/ using document root <none> and password file <none>
06:59:14.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.986488 15637 fs_manager.cc:359] Metadata directory not provided
06:59:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.986685 15637 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.987059 15637 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.992017 15637 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/data/instance:
06:59:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0fa436b291240e98a98bcec8ce72168"
06:59:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:14 on dist-test-slave-4c6b"
06:59:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:14.993141 15637 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/wal/instance:
06:59:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0fa436b291240e98a98bcec8ce72168"
06:59:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:14 on dist-test-slave-4c6b"
06:59:15.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.000319 15637 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.007s
06:59:15.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.005571 15656 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:15.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.006568 15637 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
06:59:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.006896 15637 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/wal
06:59:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0fa436b291240e98a98bcec8ce72168"
06:59:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:14 on dist-test-slave-4c6b"
06:59:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.007256 15637 fs_report.cc:383] FS layout report
06:59:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/wal
06:59:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/wal
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/data/data
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:15.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.098944 15637 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.100950 15637 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.101437 15637 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.103895 15637 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:59:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.108000 15637 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:59:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.108301 15637 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:59:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.108624 15637 ts_tablet_manager.cc:590] Registered 0 tablets
06:59:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.108791 15637 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:59:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.243126 15637 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.195:35857
06:59:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.243202 15768 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.195:35857 every 8 connection(s)
06:59:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.245625 15637 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/data/info.pb
06:59:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.253672 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15637
06:59:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.254235 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster3058788104497620504/ts-2/wal/instance
06:59:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.290143 15771 heartbeater.cc:347] Connected to a master server at 127.14.179.254:41005
06:59:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.290390 15770 heartbeater.cc:347] Connected to a master server at 127.14.179.253:37719
06:59:15.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.290735 15771 heartbeater.cc:464] Registering TS with master...
06:59:15.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.290771 15770 heartbeater.cc:464] Registering TS with master...
06:59:15.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.291509 15769 heartbeater.cc:347] Connected to a master server at 127.14.179.252:42273
06:59:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.291849 15769 heartbeater.cc:464] Registering TS with master...
06:59:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.292181 15770 heartbeater.cc:511] Master 127.14.179.253:37719 requested a full tablet report, sending...
06:59:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.292194 15771 heartbeater.cc:511] Master 127.14.179.254:41005 requested a full tablet report, sending...
06:59:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.292673 15769 heartbeater.cc:511] Master 127.14.179.252:42273 requested a full tablet report, sending...
06:59:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.295264 15254 ts_manager.cc:194] Registered new tserver with Master: f0fa436b291240e98a98bcec8ce72168 (127.14.179.195:35857)
06:59:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.295507 15100 ts_manager.cc:194] Registered new tserver with Master: f0fa436b291240e98a98bcec8ce72168 (127.14.179.195:35857)
06:59:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.295516 15174 ts_manager.cc:194] Registered new tserver with Master: f0fa436b291240e98a98bcec8ce72168 (127.14.179.195:35857)
06:59:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.297320 15254 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.179.195:59255
06:59:15.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.308013 15055 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:59:15.318 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:59:15.320 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
06:59:15.334 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.14.179.254"
port: 41005
}
bound_http_address {
host: "127.14.179.254"
port: 44527
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.14.179.253"
port: 37719
}
bound_http_address {
host: "127.14.179.253"
port: 35291
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.14.179.252"
port: 42273
}
bound_http_address {
host: "127.14.179.252"
port: 46839
}
}
}
06:59:15.338 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:59:15.341 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.14.179.193"
port: 45699
}
bound_http_address {
host: "127.14.179.193"
port: 42699
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.14.179.194"
port: 39367
}
bound_http_address {
host: "127.14.179.194"
port: 41975
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.14.179.195"
port: 35857
}
bound_http_address {
host: "127.14.179.195"
port: 41241
}
}
}
06:59:15.341 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:59:15.509 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
06:59:15.631 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskhc0eXq/test-tmp
06:59:15.632 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:59:15.632 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskhc0eXq/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:59:15.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
06:59:15.964 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:15.964 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:59:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.967492 15780 mini_oidc.cc:152] Starting JWKS server
06:59:15.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.971819 15780 webserver.cc:460] Webserver started at http://127.0.0.1:46709/ using document root <none> and password file <none>
06:59:15.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.972342 15780 mini_oidc.cc:189] Starting OIDC Discovery server
06:59:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:15.974112 15780 webserver.cc:460] Webserver started at http://127.0.0.1:35041/ using document root <none> and password file <none>
06:59:16.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
06:59:16.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/principal' for realm 'KRBTEST.COM',
06:59:16.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
06:59:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:16 dist-test-slave-4c6b krb5kdc[15792](info): setting up network...
06:59:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
06:59:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:16 dist-test-slave-4c6b krb5kdc[15792](info): set up 2 sockets
06:59:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:16 dist-test-slave-4c6b krb5kdc[15792](info): commencing operation
06:59:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
06:59:16.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:16.300889 15769 heartbeater.cc:503] Master 127.14.179.252:42273 was elected leader, sending a full tablet report...
06:59:17.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.013391 15780 mini_kdc.cc:121] Time spent starting KDC: real 1.001s user 0.004s sys 0.016s
06:59:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
06:59:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:59:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
06:59:17.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
06:59:17.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:59:17.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
06:59:17.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
06:59:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:59:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
06:59:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:59:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/test-user.keytab.
06:59:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/test-user.keytab.
06:59:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:17 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114757, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:17.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:59:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:59:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:17Z Disabled control of system clock
06:59:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.105.62@KRBTEST.COM; defaulting to no policy
06:59:17.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:59:17.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.105.62@KRBTEST.COM" created.
06:59:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:59:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.62.keytab.
06:59:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.62.keytab.
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.199215 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/logs
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/info.pb
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.62:40825
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.62
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.62.keytab
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.62
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:59:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.509488 15806 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.510038 15806 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.510648 15806 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:17.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.541790 15806 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:59:17.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.542032 15806 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:17.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.542328 15806 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:17.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.542557 15806 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:17.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.542734 15806 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:17.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.542906 15806 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:17.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.543056 15806 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46709/jwks.json
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.577867 15806 master_runner.cc:386] Master server non-default flags:
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.62.keytab
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.62
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.62:40825
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/info.pb
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.62
06:59:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/logs
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.579195 15806 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:17.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.580246 15806 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:17.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.596467 15812 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.596791 15813 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:17.597105 15815 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:17.598065 15806 server_base.cc:1034] running on GCE node
06:59:18.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.688920 15806 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:18.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.691753 15806 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:18.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.693153 15806 hybrid_clock.cc:648] HybridClock initialized: now 1714114758693131 us; error 41 us; skew 500 ppm
06:59:18.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.693996 15806 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:18.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:18 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114758, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:18.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.700034 15806 init.cc:377] Logged in from keytab as kudu/127.15.105.62@KRBTEST.COM (short username kudu)
06:59:18.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.705034 15806 webserver.cc:460] Webserver started at http://127.15.105.62:34657/ using document root <none> and password file <none>
06:59:18.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.705881 15806 fs_manager.cc:359] Metadata directory not provided
06:59:18.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.706064 15806 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:18.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.706502 15806 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.711223 15806 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/instance:
06:59:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3583a679bf3d4363a92faa7bd388acb6"
06:59:18.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:18 on dist-test-slave-4c6b"
06:59:18.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.712291 15806 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal/instance:
06:59:18.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3583a679bf3d4363a92faa7bd388acb6"
06:59:18.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:18 on dist-test-slave-4c6b"
06:59:18.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.719252 15806 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
06:59:18.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.724745 15823 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.725804 15806 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.726090 15806 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3583a679bf3d4363a92faa7bd388acb6"
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:18 on dist-test-slave-4c6b"
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.726449 15806 fs_report.cc:383] FS layout report
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/data
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.792903 15806 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.798183 15806 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.798669 15806 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.865528 15806 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.62:40825
06:59:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.865607 15874 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.62:40825 every 8 connection(s)
06:59:18.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.868204 15806 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/info.pb
06:59:18.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.869886 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15806
06:59:18.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.870496 15780 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal/instance
06:59:18.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.873873 15875 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.105.61@KRBTEST.COM; defaulting to no policy
06:59:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.884685 15875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } has no permanent_uuid. Determining permanent_uuid...
06:59:18.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:59:18.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.105.61@KRBTEST.COM" created.
06:59:18.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:18 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114758, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.62@KRBTEST.COM for kudu/127.15.105.62@KRBTEST.COM
06:59:18.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:59:18.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.61.keytab.
06:59:18.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.61.keytab.
06:59:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.915270 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data
06:59:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/logs
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/info.pb
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.61:41047
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.61
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.61.keytab
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.61
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:59:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.924188 15875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } has no permanent_uuid. Determining permanent_uuid...
06:59:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:18.926362 15826 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:18.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:18.930285 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41047: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:18.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:18.979797 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } attempt: 1
06:59:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:18.983634 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41047: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.051980 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } attempt: 2
06:59:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.055779 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41047: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.147243 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } attempt: 3
06:59:19.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.152321 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41047: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.220274 15881 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.220791 15881 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.221349 15881 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:19.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.251224 15881 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:59:19.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.251456 15881 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.251996 15881 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.252303 15881 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.252745 15881 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.253103 15881 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.253362 15881 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46709/jwks.json
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.289543 15881 master_runner.cc:386] Master server non-default flags:
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.61.keytab
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.61
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.61:41047
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/info.pb
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.61
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/logs
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.290745 15881 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:19.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.291838 15881 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.295730 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } attempt: 4
06:59:19.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.300753 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41047: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.310734 15890 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:19.600190 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } attempt: 5
06:59:19.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.608384 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41047: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.155989 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } attempt: 6
06:59:20.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:20.187403 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41047: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:20.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:19.312556 15889 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:20.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:20.560946 15891 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1248 milliseconds
06:59:20.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:20.561807 15892 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:20.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:20.564599 15881 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.255s user 0.364s sys 0.873s
06:59:20.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:20.564855 15881 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.256s user 0.364s sys 0.873s
06:59:20.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.565029 15881 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:59:20.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.566093 15881 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:20.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.568894 15881 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.570283 15881 hybrid_clock.cc:648] HybridClock initialized: now 1714114760570254 us; error 39 us; skew 500 ppm
06:59:20.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.571084 15881 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:20 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114760, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.584662 15881 init.cc:377] Logged in from keytab as kudu/127.15.105.61@KRBTEST.COM (short username kudu)
06:59:20.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.590284 15881 webserver.cc:460] Webserver started at http://127.15.105.61:39765/ using document root <none> and password file <none>
06:59:20.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.591511 15881 fs_manager.cc:359] Metadata directory not provided
06:59:20.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.591769 15881 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:20.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.592185 15881 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.600119 15881 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/instance:
06:59:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83796ee80daa4f1cbadcd4bc74256641"
06:59:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:20 on dist-test-slave-4c6b"
06:59:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.601785 15881 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal/instance:
06:59:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83796ee80daa4f1cbadcd4bc74256641"
06:59:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:20 on dist-test-slave-4c6b"
06:59:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.612002 15881 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.003s
06:59:20.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.618952 15902 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.620029 15881 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
06:59:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.620333 15881 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83796ee80daa4f1cbadcd4bc74256641"
06:59:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:20 on dist-test-slave-4c6b"
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.620625 15881 fs_report.cc:383] FS layout report
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/data
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:20.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.699388 15881 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:20.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.704887 15881 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:20.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.705369 15881 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.786060 15881 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.61:41047
06:59:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.786170 15953 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.61:41047 every 8 connection(s)
06:59:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.788791 15881 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/info.pb
06:59:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.793776 15954 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:20.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.796494 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15881
06:59:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.796939 15780 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal/instance
06:59:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.806939 15954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } has no permanent_uuid. Determining permanent_uuid...
06:59:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.105.60@KRBTEST.COM; defaulting to no policy
06:59:20.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:59:20.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.105.60@KRBTEST.COM" created.
06:59:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:20 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114760, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.61@KRBTEST.COM for kudu/127.15.105.62@KRBTEST.COM
06:59:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:59:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.60.keytab.
06:59:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.60.keytab.
06:59:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.838732 15954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } has no permanent_uuid. Determining permanent_uuid...
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.841662 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/logs
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/info.pb
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.60:46185
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.60
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.60.keytab
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.60
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:59:20.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:20.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:59:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:20 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114760, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.61@KRBTEST.COM for kudu/127.15.105.61@KRBTEST.COM
06:59:20.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.863564 15954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } has no permanent_uuid. Determining permanent_uuid...
06:59:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:20.865718 15905 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:20.877233 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.926779 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 1
06:59:20.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:20.930727 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:20.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:20.999099 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 2
06:59:21.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.002851 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.094204 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 3
06:59:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.097769 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.159519 15961 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.160079 15961 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.160645 15961 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.188838 15961 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:59:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.189087 15961 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.189353 15961 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.189532 15961 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.189730 15961 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.189913 15961 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.190052 15961 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46709/jwks.json
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.224879 15961 master_runner.cc:386] Master server non-default flags:
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.60.keytab
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.60
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.60:46185
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/info.pb
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.60
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/logs
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.225977 15961 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:21.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.226986 15961 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:21.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.241205 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 4
06:59:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.242673 15969 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.246680 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.247975 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } attempt: 7
06:59:21.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:21 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114758, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.62@KRBTEST.COM for kudu/127.15.105.61@KRBTEST.COM
06:59:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.284320 15875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } has no permanent_uuid. Determining permanent_uuid...
06:59:21.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.293118 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.351670 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 1
06:59:21.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.359932 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.441592 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 2
06:59:21.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.449501 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.535018 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 3
06:59:21.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.543862 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.546211 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 5
06:59:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.552117 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.684298 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 4
06:59:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.687912 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:21.971354 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 5
06:59:21.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.978611 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.099493 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 6
06:59:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:22.120828 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:22.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.531148 15875 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } attempt: 6
06:59:22.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.244000 15968 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:22.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:21.243326 15971 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:22.537716 15970 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1292 milliseconds
06:59:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.537853 15961 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:59:22.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.539230 15961 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:22.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:22.539356 15875 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:46185: Network error: Client connection negotiation failed: client connection to 127.15.105.60:46185: connect: Connection refused (error 111)
06:59:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.542016 15961 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.543356 15961 hybrid_clock.cc:648] HybridClock initialized: now 1714114762543327 us; error 41 us; skew 500 ppm
06:59:22.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.544166 15961 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:22 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114762, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.549846 15961 init.cc:377] Logged in from keytab as kudu/127.15.105.60@KRBTEST.COM (short username kudu)
06:59:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.555591 15961 webserver.cc:460] Webserver started at http://127.15.105.60:41725/ using document root <none> and password file <none>
06:59:22.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.556632 15961 fs_manager.cc:359] Metadata directory not provided
06:59:22.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.556893 15961 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:22.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.557457 15961 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:59:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:22.563166 15206 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9fa0 after lost signal to thread 15141
06:59:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.563484 15961 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/instance:
06:59:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:22.563748 15206 debug-util.cc:398] Leaking SignalData structure 0x7b0800099320 after lost signal to thread 15209
06:59:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94be02d238a44c65bda90b541221c334"
06:59:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:22 on dist-test-slave-4c6b"
06:59:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.564677 15961 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal/instance:
06:59:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94be02d238a44c65bda90b541221c334"
06:59:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:22 on dist-test-slave-4c6b"
06:59:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.572141 15961 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
06:59:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.577862 15986 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.578935 15961 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:59:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.579293 15961 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94be02d238a44c65bda90b541221c334"
06:59:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:22 on dist-test-slave-4c6b"
06:59:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.579640 15961 fs_report.cc:383] FS layout report
06:59:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/data
06:59:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:22.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.653857 15961 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:22.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.660660 15961 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:22.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.661196 15961 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.753332 15961 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.60:46185
06:59:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.753489 16037 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.60:46185 every 8 connection(s)
06:59:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.756055 15961 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/info.pb
06:59:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.757737 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 15961
06:59:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.758308 15780 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal/instance
06:59:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.762658 16038 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:59:22.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:22 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114757, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.105.60@KRBTEST.COM
06:59:22.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.813805 16038 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } has no permanent_uuid. Determining permanent_uuid...
06:59:22.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:22 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114762, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for kudu/127.15.105.62@KRBTEST.COM
06:59:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:22 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114757, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.105.61@KRBTEST.COM
06:59:22.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.861304 16038 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } has no permanent_uuid. Determining permanent_uuid...
06:59:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:22 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114762, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for kudu/127.15.105.61@KRBTEST.COM
06:59:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:22 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114757, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.105.62@KRBTEST.COM
06:59:22.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.902458 16038 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } has no permanent_uuid. Determining permanent_uuid...
06:59:22.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.909406 15780 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
06:59:22.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:22 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114762, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for kudu/127.15.105.60@KRBTEST.COM
06:59:22.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:22.921 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
06:59:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.938886 16038 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: Bootstrap starting.
06:59:22.940 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
06:59:22.940 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
06:59:22.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.105.62"
port: 40825
}
bound_http_address {
host: "127.15.105.62"
port: 34657
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.105.61"
port: 41047
}
bound_http_address {
host: "127.15.105.61"
port: 39765
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.105.60"
port: 46185
}
bound_http_address {
host: "127.15.105.60"
port: 41725
}
}
}
06:59:22.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:59:22.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:22.945 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:59:22.946 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:59:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.946691 16038 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: Neither blocks nor log segments found. Creating new log.
06:59:22.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.950410 16038 log.cc:824] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: Log is configured to *not* fsync() on all Append() calls
06:59:22.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.961740 16038 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: No bootstrap required, opened a new log
06:59:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:22 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114762, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:22.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:59:22.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:22.993 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc
06:59:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.995136 16038 raft_consensus.cc:348] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.996034 16038 raft_consensus.cc:374] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.996337 16038 raft_consensus.cc:729] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 94be02d238a44c65bda90b541221c334, State: Initialized, Role: FOLLOWER
06:59:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:22.997267 16038 consensus_queue.cc:260] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.000473 16050 sys_catalog.cc:455] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } } }
06:59:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.001524 16050 sys_catalog.cc:458] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [sys.catalog]: This master's current role is: FOLLOWER
06:59:23.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.002444 16038 sys_catalog.cc:564] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [sys.catalog]: configured and running, proceeding with master startup.
06:59:23.003 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:59:23.003 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc
06:59:23.005 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:59:23.009 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
06:59:23.011 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:59:23.011 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:59:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:23 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114763, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:59:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.049185 16062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.049543 16062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:23.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:23.054 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc
06:59:23.055 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:59:23.056 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.105.60:46185
06:59:23.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
06:59:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.061142 15780 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15961
06:59:23.082 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:23.083 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.105.62:40825
06:59:23.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
06:59:23.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.083882 15780 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15806
06:59:23.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:23.111 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.105.61:41047
06:59:23.112 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
06:59:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.112610 15780 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15881
06:59:23.134 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:23.134 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.105.60:46185
06:59:23.135 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.137256 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/logs
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/info.pb
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.60:46185
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.60
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41725
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.60.keytab
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.60
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:59:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:59:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.451083 16065 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.451637 16065 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:23.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.452309 16065 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.483559 16065 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:59:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.483814 16065 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.484192 16065 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.484409 16065 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.484601 16065 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:23.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.484786 16065 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.484925 16065 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46709/jwks.json
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.519596 16065 master_runner.cc:386] Master server non-default flags:
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.60.keytab
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.60
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.60:46185
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/info.pb
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.60
06:59:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41725
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/logs
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.520766 16065 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.521706 16065 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:23.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.538497 16072 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:24.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.538900 16073 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:24.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:23.541416 16065 server_base.cc:1034] running on GCE node
06:59:24.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:23.539312 16075 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.667706 16065 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.670096 16065 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.671423 16065 hybrid_clock.cc:648] HybridClock initialized: now 1714114764671402 us; error 41 us; skew 500 ppm
06:59:24.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.672175 16065 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:24.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:24 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114764, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:24.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.677654 16065 init.cc:377] Logged in from keytab as kudu/127.15.105.60@KRBTEST.COM (short username kudu)
06:59:24.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.682543 16065 webserver.cc:460] Webserver started at http://127.15.105.60:41725/ using document root <none> and password file <none>
06:59:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.683333 16065 fs_manager.cc:359] Metadata directory not provided
06:59:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.683518 16065 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:24.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.690682 16065 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
06:59:24.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.694600 16084 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:24.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.695557 16065 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
06:59:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.695828 16065 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94be02d238a44c65bda90b541221c334"
06:59:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:22 on dist-test-slave-4c6b"
06:59:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.698163 16065 fs_report.cc:383] FS layout report
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/wal
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/data
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.763604 16065 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.769129 16065 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.769521 16065 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.836652 16065 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.60:46185
06:59:24.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.836771 16135 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.60:46185 every 8 connection(s)
06:59:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.839340 16065 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-2/data/info.pb
06:59:24.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.848778 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 16065
06:59:24.849 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:24.849 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.105.62:40825
06:59:24.849 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
06:59:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.850246 16136 sys_catalog.cc:263] Verifying existing consensus state
06:59:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.851480 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/logs
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/info.pb
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.62:40825
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.62
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34657
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.62.keytab
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.62
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:59:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.858799 16136 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: Bootstrap starting.
06:59:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.890735 16136 log.cc:824] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: Log is configured to *not* fsync() on all Append() calls
06:59:24.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.895457 16136 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:59:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.896088 16136 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: Bootstrap complete.
06:59:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.919920 16136 raft_consensus.cc:348] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.922260 16136 raft_consensus.cc:374] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:59:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.922506 16136 raft_consensus.cc:729] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 94be02d238a44c65bda90b541221c334, State: Initialized, Role: FOLLOWER
06:59:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.923259 16136 consensus_queue.cc:260] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.926151 16140 sys_catalog.cc:455] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } } }
06:59:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.927103 16140 sys_catalog.cc:458] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [sys.catalog]: This master's current role is: FOLLOWER
06:59:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.927874 16136 sys_catalog.cc:564] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [sys.catalog]: configured and running, proceeding with master startup.
06:59:24.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:24.970623 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:24.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:24.971307 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.955780 16065 master_runner.cc:186] Error getting master registration for 127.15.105.62:40825: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111),
06:59:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:24.989570 16065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:25.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.026827 16090 tablet.cc:2369] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:59:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:25.083451 16140 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:25.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:25.084151 16140 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:25.089588 16140 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:25.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.090040 16088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.104760 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.107403 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:25.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:25.108003 16088 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:25.108947 16140 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:25.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.265441 16138 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:25.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.266260 16138 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:25.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.267339 16138 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.320775 16138 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:59:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.321116 16138 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.321556 16138 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.321898 16138 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.322201 16138 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.322532 16138 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.322772 16138 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46709/jwks.json
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:25.383566 16138 master_runner.cc:386] Master server non-default flags:
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.62.keytab
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.62
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.62:40825
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/info.pb
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.62
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34657
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/logs
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:25.385100 16138 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:25.386430 16138 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.404291 16162 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:25.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.977931 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:25.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.978410 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:26.867358 16170 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:26.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:26.867973 16170 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:26.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:26.881479 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:26.885609 16170 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:26.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:26.887944 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:26.891175 16088 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:26.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:26.892166 16170 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:26.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:26.984323 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:26.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:26.984774 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:26.805346 16161 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 16138
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:27.432528 16161 kernel_stack_watchdog.cc:198] Thread 16138 stuck at /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:637 for 397ms:
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:25.405707 16163 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:27.434279 16138 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.029s user 0.838s sys 1.155s
06:59:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:27.434612 16138 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.029s user 0.838s sys 1.155s
06:59:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:27.435461 16165 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:27.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:27.439119 16164 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:59:27.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.439141 16138 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:59:27.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.440738 16138 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:27.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.443614 16138 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:27.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.445031 16138 hybrid_clock.cc:648] HybridClock initialized: now 1714114767444937 us; error 103 us; skew 500 ppm
06:59:27.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.445858 16138 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:27.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:27 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114767, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:27.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.451777 16138 init.cc:377] Logged in from keytab as kudu/127.15.105.62@KRBTEST.COM (short username kudu)
06:59:27.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.457195 16138 webserver.cc:460] Webserver started at http://127.15.105.62:34657/ using document root <none> and password file <none>
06:59:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.458448 16138 fs_manager.cc:359] Metadata directory not provided
06:59:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.458736 16138 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.469406 16138 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
06:59:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.473954 16179 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:27.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.475147 16138 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:59:27.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.475524 16138 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:27.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3583a679bf3d4363a92faa7bd388acb6"
06:59:27.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:18 on dist-test-slave-4c6b"
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.478708 16138 fs_report.cc:383] FS layout report
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/data
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.535439 16138 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.541694 16138 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.542207 16138 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.630266 16138 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.62:40825
06:59:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.630350 16230 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.62:40825 every 8 connection(s)
06:59:27.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.633323 16138 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/data/info.pb
06:59:27.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:27.638 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.105.61:41047
06:59:27.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
06:59:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.635211 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 16138
06:59:27.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.641273 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/logs
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/info.pb
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.61:41047
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.61
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39765
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.61.keytab
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.61
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:59:27.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.669867 16231 sys_catalog.cc:263] Verifying existing consensus state
06:59:27.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240426 06:59:27.670773 16231 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:59:27.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.671304 16138 master.cc:537] Master@127.15.105.62:40825 shutting down...
06:59:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:27.763039 16138 master.cc:552] Master@127.15.105.62:40825 shutdown complete.
06:59:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:59:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:27.989184 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:27.989490 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:28.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.008100 16232 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:59:28.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.008654 16232 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:59:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.009351 16232 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:59:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.039695 16232 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:59:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.039950 16232 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:59:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.040370 16232 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:59:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.040704 16232 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:59:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.041033 16232 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:59:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.041355 16232 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:59:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.041564 16232 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46709/jwks.json
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:28.076784 16232 master_runner.cc:386] Master server non-default flags:
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:36471
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:40825,127.15.105.61:41047,127.15.105.60:46185
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/krb5kdc/kudu_127.15.105.61.keytab
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.61
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:59:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:59:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.61:41047
06:59:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:59:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:59:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:59:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46709/jwks.json
06:59:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/info.pb
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.61
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39765
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/logs
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:28.078325 16232 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:59:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:28.079684 16232 file_cache.cc:492] Constructed file cache file cache with capacity 419430
06:59:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.097033 16240 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:28.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:28.669538 16245 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:28.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:28.669898 16245 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:28.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:28.671244 16245 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.677666 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.678891 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:28.679569 16088 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:28.680516 16245 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:28.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.993149 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:28.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.993436 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.097740 16242 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:29.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:28.099020 16239 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:59:29.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:29.221283 16241 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:59:29.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.221369 16232 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:59:29.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.226018 16232 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:59:29.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.229099 16232 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:59:29.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.230450 16232 hybrid_clock.cc:648] HybridClock initialized: now 1714114769230416 us; error 48 us; skew 500 ppm
06:59:29.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.231174 16232 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:59:29.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:29 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114769, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:29.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.236937 16232 init.cc:377] Logged in from keytab as kudu/127.15.105.61@KRBTEST.COM (short username kudu)
06:59:29.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.243103 16232 webserver.cc:460] Webserver started at http://127.15.105.61:39765/ using document root <none> and password file <none>
06:59:29.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.244256 16232 fs_manager.cc:359] Metadata directory not provided
06:59:29.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.244526 16232 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:59:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.253367 16232 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.002s
06:59:29.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.257709 16255 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:59:29.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.258819 16232 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:59:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.259157 16232 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data,/tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83796ee80daa4f1cbadcd4bc74256641"
06:59:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 06:59:20 on dist-test-slave-4c6b"
06:59:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.261415 16232 fs_report.cc:383] FS layout report
06:59:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/data
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:59:29.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:59:29.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.315407 16232 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
06:59:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.321373 16232 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:59:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.321753 16232 kserver.cc:163] Server-wide thread pool size limit: 3276
06:59:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.393510 16232 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.61:41047
06:59:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.393592 16306 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.61:41047 every 8 connection(s)
06:59:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.396308 16232 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/data/info.pb
06:59:29.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.403240 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu as pid 16232
06:59:29.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:59:29.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.407083 16307 sys_catalog.cc:263] Verifying existing consensus state
06:59:29.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240426 06:59:29.407943 16307 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:59:29.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.408443 16232 master.cc:537] Master@127.15.105.61:41047 shutting down...
06:59:29.458 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.105.60:46185(127.15.105.60:46185)] connecting to peer
06:59:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.483117 16232 master.cc:552] Master@127.15.105.61:41047 shutdown complete.
06:59:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskhc0eXq/test-tmp/mini-kudu-cluster7876240971382427793/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:59:29.515 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:29.517 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:29.571 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.105.60:46185(127.15.105.60:46185)] Successfully connected to peer
06:59:29.573 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.574 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.627 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.641 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
06:59:29.650 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.105.60
06:59:29.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:29 dist-test-slave-4c6b krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114763, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.105.60@KRBTEST.COM
06:59:29.855 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7ce880dc, L:/127.0.0.1:51904 - R:/127.15.105.60:46185] using SASL/GSSAPI
06:59:29.869 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:29.891 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:29.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:29.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.897 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.899 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:29.908 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:29.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:29.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.912 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.912 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.913 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.914 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:29.929 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:29.930 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:29.933 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.933 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.934 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.934 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.935 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.935 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:29.969 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:29.969 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:29.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.974 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.974 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.975 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.975 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:29.989 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:29.990 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:29.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.992848 16065 master_runner.cc:186] Error getting master registration for 127.15.105.62:40825: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111),
06:59:29.995 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.995 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:29.993278 16065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:29.995 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:29.997 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.997 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:29.997116 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:29.997426 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:29.997 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:29.998 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:30.049 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:30.052 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:30.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.055 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.055 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.056 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.056 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:30.169 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:30.171 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:30.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.172 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.174 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.174 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.175 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.175 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:30.429 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:30.431 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:30.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.433 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.435 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.436 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.437 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.437 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:30.469 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:30.470 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:30.472 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.472 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.472 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.473 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.473 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.474 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:30.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:30.550 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:30.551 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.553 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:30.729 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:30.730 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:30.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:30.734 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.734 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.734 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:30.734 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:31.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:31.001909 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:31.002265 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:32.000253 15285 debug-util.cc:398] Leaking SignalData structure 0x7b08000a3380 after lost signal to thread 15217
06:59:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:32.000813 15285 debug-util.cc:398] Leaking SignalData structure 0x7b080009e320 after lost signal to thread 15288
06:59:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:32.005676 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:32.005901 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:32.082648 16316 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:32.083019 16316 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:32.084421 16316 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:32.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:32.087498 16088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111) [suppressed 5 similar messages]
06:59:32.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:32.090431 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:32.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:32.091593 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:32.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:32.092039 16087 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:32.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:32.092756 16316 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:33.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:33.008952 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:33.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:33.009294 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:33.109 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:33.110 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:33.113 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:33.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:33.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:33.114 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:33.115 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:33.115 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:33.115 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:34.012837 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:34.013165 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:34 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114774, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:34.681317 16080 init.cc:295] Successfully reacquired a new kerberos TGT
06:59:34.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:34.910 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:34.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:34.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:34.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:34.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:34.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:34.913 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:34.914 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:34.996589 16065 master_runner.cc:186] Error getting master registration for 127.15.105.62:40825: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111),
06:59:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:34.997112 16065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:35.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:35.016134 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:35.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:35.016386 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:36.019639 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:36.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:36.019907 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:36.328621 16327 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:36.329094 16327 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:36.330674 16327 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:36.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:36.335235 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:36.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:36.335767 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:36.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:36.336061 16087 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:36.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:36.336618 16327 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:37.022663 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:37.022929 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:37.709 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:37.711 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:37.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:37.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:37.713 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:37.714 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:37.714 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:37.714 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:37.715 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:38.025701 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:38.025967 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:39.028862 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:39.029202 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:39.999725 16065 master_runner.cc:186] Error getting master registration for 127.15.105.62:40825: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111),
06:59:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:40.000038 16065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:40.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:40.032028 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:40.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:40.032285 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:41.035107 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:41.035353 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:41.349 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:41.350 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:41.352 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:41.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:41.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:41.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:41.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:41.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:41.354 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:41.769 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:41.770 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:41.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:41.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:41.772 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:41.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:41.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:41.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:41.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:42.038098 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:42.038383 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:43.041087 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:43.041329 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:43.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:43.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:43.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:43.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:43.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:43.673 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:43.673 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:43.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:43.674 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:44.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:44.044564 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:44.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:44.044857 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:44 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114784, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:44.684203 16080 init.cc:295] Successfully reacquired a new kerberos TGT
06:59:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:45.002897 16065 master_runner.cc:186] Error getting master registration for 127.15.105.62:40825: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111),
06:59:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:45.003248 16065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:45.047591 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:45.047829 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:45.509 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:45.510 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:45.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:45.511 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:45.511 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:45.512 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:45.512 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:45.512 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:45.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:45.537869 16341 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:45.538394 16341 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:45.540273 16341 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:45.542399 16088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111) [suppressed 3 similar messages]
06:59:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:45.544813 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:45.545342 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:45.545605 16087 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:45.546090 16341 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:45.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:45.699172 15206 debug-util.cc:398] Leaking SignalData structure 0x7b0800077ae0 after lost signal to thread 15141
06:59:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:45.700203 15206 debug-util.cc:398] Leaking SignalData structure 0x7b0800091f40 after lost signal to thread 15209
06:59:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:46.050691 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:46.050958 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:47.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:47.053640 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:47.053905 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:47.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:47.271 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:47.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:47.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:47.273 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:47.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:47.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:47.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:47.274 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:47.830 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:47.830 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:47.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:47.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:47.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:47.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:47.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:47.832 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:47.833 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:48.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:48.056826 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:48.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:48.057080 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:48.336055 16347 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:48.336454 16347 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:48.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:48.338073 16347 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:48.342484 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:48.342846 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:48.343132 16087 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:48.343637 16347 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:48.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:48.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:48.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:48.690 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:48.690 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:48.690 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:48.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:48.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:48.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:49.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.059924 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:49.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:49.060192 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.005821 16065 master_runner.cc:186] Error getting master registration for 127.15.105.62:40825: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111),
06:59:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:50.006151 16065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.062976 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:50.063211 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:51.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:51.066078 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:51.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:51.066325 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:52.069139 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:52.069440 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:52.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:52.550 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:52.550 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:52.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:52.551 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:52.551 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:52.551 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:52.551 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:52.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:53.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:53.072129 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:53.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:53.072357 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:53.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.931841 16358 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.932391 16358 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.934159 16358 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:53.936589 16087 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111) [suppressed 3 similar messages]
06:59:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:53.939145 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:53.939579 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.939888 16087 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:53.940452 16358 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:54.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:54.075688 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:54.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:54.075986 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:54.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 06:59:54 dist-test-slave-4c6b krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714114794, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:59:54.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:54.686331 16080 init.cc:295] Successfully reacquired a new kerberos TGT
06:59:55.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:55.008728 16065 master_runner.cc:186] Error getting master registration for 127.15.105.62:40825: Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111),
06:59:55.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:55.009080 16065 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:59:55.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:55.078644 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:55.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:55.078864 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:55.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:55.486373 15484 debug-util.cc:398] Leaking SignalData structure 0x7b08000b60e0 after lost signal to thread 15355
06:59:55.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:55.487139 15484 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0ae0 after lost signal to thread 15487
06:59:55.989 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:55.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:55.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:55.991 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:55.992 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:55.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:55.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:55.994 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:55.994 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:56.081251 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:56.081485 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:56.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:56.784247 16132 debug-util.cc:398] Leaking SignalData structure 0x7b080009a260 after lost signal to thread 16067
06:59:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:56.784852 16132 debug-util.cc:398] Leaking SignalData structure 0x7b08000981c0 after lost signal to thread 16135
06:59:57.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:57.083789 16151 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:59:57.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:57.084004 16151 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:59:57.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:57.669437 16366 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:59:57.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:57.669816 16366 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3583a679bf3d4363a92faa7bd388acb6" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 40825 } } peers { permanent_uuid: "83796ee80daa4f1cbadcd4bc74256641" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41047 } } peers { permanent_uuid: "94be02d238a44c65bda90b541221c334" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 46185 } }
06:59:57.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:57.671304 16366 leader_election.cc:288] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825), 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047)
06:59:57.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:57.677067 16088 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3583a679bf3d4363a92faa7bd388acb6 (127.15.105.62:40825): Network error: Client connection negotiation failed: client connection to 127.15.105.62:40825: connect: Connection refused (error 111)
06:59:57.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 06:59:57.677541 16087 leader_election.cc:334] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 83796ee80daa4f1cbadcd4bc74256641 (127.15.105.61:41047): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41047: connect: Connection refused (error 111)
06:59:57.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:57.677912 16087 leader_election.cc:302] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94be02d238a44c65bda90b541221c334; no voters: 3583a679bf3d4363a92faa7bd388acb6, 83796ee80daa4f1cbadcd4bc74256641
06:59:57.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:57.678529 16366 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94be02d238a44c65bda90b541221c334 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:59:57.949 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:40825(127.15.105.62:40825)] connecting to peer
06:59:57.950 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41047(127.15.105.61:41047)] connecting to peer
06:59:57.950 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:57.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:40825(127.15.105.62:40825)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:57.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:40825: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825
06:59:57.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:57.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41047(127.15.105.61:41047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:57.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41047: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
06:59:57.956 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047; will retry
06:59:57.964 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28549), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [162ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [214ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [453ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [458ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [483ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [485ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [486ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [487ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [499ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [500ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [501ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [521ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [524ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [560ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [561ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [563ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [583ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [584ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [586ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [640ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [641ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [644ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [759ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [761ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [762ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [763ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1025ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1063ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1136ms] refreshing cache from master, [1136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1141ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1323ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [3696ms] refreshing cache from master, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [3698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [3701ms] too many traces: truncated at 100 traces, deferred=Deferred@1387641605(state=PENDING, result=null, callback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187), errback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187)))
06:59:57.969 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.105.61:41047(127.15.105.61:41047)] disconnecting while in state TERMINATED
06:59:57.970 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.105.60:46185(127.15.105.60:46185)] disconnecting while in state READY
06:59:57.970 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.105.62:40825(127.15.105.62:40825)] disconnecting while in state TERMINATED
06:59:57.976 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:59:57.979 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:59:57.980 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.105.60:46185(127.15.105.60:46185)] handling channelInactive
06:59:57.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.60:46185(127.15.105.60:46185)] cleaning up while in state READY due to: connection closed
06:59:57.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 06:59:57.991895 15780 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 16065
06:59:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T06:59:58Z chronyd exiting
07:00:00.042 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:00:00.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:00.043334 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15334
07:00:00.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:00.067970 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15492
07:00:00.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:00.092401 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15637
07:00:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:00.117313 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15067
07:00:00.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:00.144258 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15137
07:00:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:00:00.171545 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhc0eXq/build/tsan/bin/kudu with pid 15212
07:00:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:00:01Z chronyd exiting
07:00:02.208 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28549), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [162ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [214ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [453ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [458ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [483ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [485ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [486ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [487ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [499ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [500ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [501ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [521ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [524ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [560ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [561ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [563ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [583ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [584ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [586ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [640ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [641ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [644ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [759ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [761ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [762ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [763ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1025ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1063ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1136ms] refreshing cache from master, [1136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1141ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1323ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [3696ms] refreshing cache from master, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [3698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [3701ms] too many traces: truncated at 100 traces, deferred=Deferred@1387641605(state=PENDING, result=null, callback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187), errback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.105.61:41047
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28549), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [162ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [214ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [453ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [458ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [483ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [485ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [486ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [487ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [499ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [500ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [501ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [521ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [524ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [560ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [561ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [563ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [583ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [584ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [586ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [640ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [641ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [644ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [759ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [761ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [762ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [763ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1025ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1063ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1136ms] refreshing cache from master, [1136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1141ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1323ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [3696ms] refreshing cache from master, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [3698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [3701ms] too many traces: truncated at 100 traces, deferred=Deferred@1387641605(state=PENDING, result=null, callback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187), errback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.105.61:41047
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
07:00:02.621 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
07:00:02.621 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28549), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [162ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [214ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [453ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [458ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [483ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [485ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [486ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [487ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [499ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [500ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [501ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [521ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [524ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [560ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [561ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [563ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [583ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [584ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [586ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [640ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [641ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [644ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [759ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [761ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [762ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [763ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1025ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1063ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1136ms] refreshing cache from master, [1136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1141ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1323ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [3696ms] refreshing cache from master, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [3698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [3701ms] too many traces: truncated at 100 traces, deferred=Deferred@1387641605(state=PENDING, result=null, callback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187), errback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.105.61:41047
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28549), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [162ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [214ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [453ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [458ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [476ms] refreshing cache from master, [477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [480ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [483ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [485ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [486ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [487ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [496ms] refreshing cache from master, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [499ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [500ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [501ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [517ms] refreshing cache from master, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [521ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [524ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [560ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [561ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [563ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [583ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [584ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [586ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [640ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [641ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [644ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [759ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [761ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [762ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [763ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1016ms] refreshing cache from master, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1025ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1057ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1063ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1136ms] refreshing cache from master, [1136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1141ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1316ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:40825: Network error: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:46185: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41047: Network error: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [1323ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047, [3696ms] refreshing cache from master, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:46185, [3697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:40825, [3698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41047, [3701ms] too many traces: truncated at 100 traces, deferred=Deferred@1387641605(state=PENDING, result=null, callback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187), errback=(continuation of Deferred@1981362900 after retry RPC after error@249328411) -> (continuation of Deferred@1999057346 after retry RPC after error@146129328) -> (continuation of Deferred@931515872 after retry RPC after error@1512842400) -> (continuation of Deferred@348894982 after retry RPC after error@844455022) -> (continuation of Deferred@732327177 after retry RPC after error@269768185) -> (continuation of Deferred@1504617338 after retry RPC after error@1090527757) -> (continuation of Deferred@1498861825 after retry RPC after error@1801452623) -> (continuation of Deferred@1195779885 after retry RPC after error@127300994) -> (continuation of Deferred@1755800232 after retry RPC after error@2081314423) -> (continuation of Deferred@168257940 after retry RPC after error@1040791347) -> (continuation of Deferred@1843961976 after retry RPC after error@1666973276) -> (continuation of Deferred@1377923178 after retry RPC after error@635856306) -> (continuation of Deferred@220588566 after retry RPC after error@825253104) -> (continuation of Deferred@1758185628 after retry RPC after error@2115431362) -> (continuation of Deferred@1547819852 after retry RPC after error@1873136616) -> (continuation of Deferred@1503424936 after retry RPC after error@278776226) -> (continuation of Deferred@1121250118 after retry RPC after error@66028159) -> (continuation of Deferred@960799387 after retry RPC after error@1198177480) -> (continuation of Deferred@578937272 after retry RPC after error@1560437741) -> (continuation of Deferred@1104036524 after retry RPC after error@1397762721) -> (continuation of Deferred@2129763666 after retry RPC after error@815961194) -> (continuation of Deferred@63601648 after retry RPC after error@1369267210) -> (continuation of Deferred@603209830 after retry RPC after error@122782134) -> (continuation of Deferred@1460961111 after retry RPC after error@1825707187)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.105.60:46185,127.15.105.62:40825,127.15.105.61:41047) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:40825(127.15.105.62:40825): Connection refused: /127.15.105.62:40825,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41047(127.15.105.61:41047): Connection refused: /127.15.105.61:41047
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.105.61:41047
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)