08:15:49.738 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:15:49.750 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0YutnF/test-tmp
08:15:49.752 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:15:49.756 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0YutnF/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:15:49.986 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:15:50.136 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:15:50.137 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:15:50.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.141008 4109 mini_oidc.cc:152] Starting JWKS server
08:15:50.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.144799 4109 webserver.cc:460] Webserver started at http://127.0.0.1:37137/ using document root <none> and password file <none>
08:15:50.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.145243 4109 mini_oidc.cc:189] Starting OIDC Discovery server
08:15:50.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.147437 4109 webserver.cc:460] Webserver started at http://127.0.0.1:44553/ using document root <none> and password file <none>
08:15:50.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:15:50Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:15:50.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:15:50Z Disabled control of system clock
08:15:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.210606 4109 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/wal
08:15:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/data
08:15:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:15:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:15:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:15:50.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/logs
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/data/info.pb
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:15:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.126:33713
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.126
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:15:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:15:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:15:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37137/jwks.json with env {}
08:15:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.518126 4121 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:15:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.518731 4121 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:15:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.519492 4121 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:15:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.551532 4121 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:15:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.551846 4121 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:15:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.552043 4121 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:15:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.552313 4121 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:15:50.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.552613 4121 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:15:50.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.552836 4121 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37137/jwks.json
08:15:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.588989 4121 master_runner.cc:386] Master server non-default flags:
08:15:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:15:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/data
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/wal
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.126:33713
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37137/jwks.json
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/data/info.pb
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.126
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/logs
08:15:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:15:50.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.590474 4121 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:15:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.592715 4121 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:15:50.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.612951 4127 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:51.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.613560 4130 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:51.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:50.616012 4121 server_base.cc:1034] running on GCE node
08:15:51.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:50.614202 4128 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.715092 4121 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:15:51.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.718175 4121 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:15:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.719579 4121 hybrid_clock.cc:648] HybridClock initialized: now 1714551351719534 us; error 58 us; skew 500 ppm
08:15:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.720520 4121 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:15:51.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.727046 4121 webserver.cc:460] Webserver started at http://127.4.3.126:41663/ using document root <none> and password file <none>
08:15:51.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.728185 4121 fs_manager.cc:359] Metadata directory not provided
08:15:51.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.728420 4121 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:15:51.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.728968 4121 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:15:51.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.734138 4121 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/data/instance:
08:15:51.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7"
08:15:51.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:51 on dist-test-slave-60sj"
08:15:51.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.735462 4121 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/wal/instance:
08:15:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7"
08:15:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:51 on dist-test-slave-60sj"
08:15:51.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.743014 4121 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
08:15:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.749104 4137 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:15:51.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.750334 4121 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:15:51.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.750703 4121 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/wal
08:15:51.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7"
08:15:51.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:51 on dist-test-slave-60sj"
08:15:51.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.751159 4121 fs_report.cc:383] FS layout report
08:15:51.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:15:51.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/wal
08:15:51.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/wal
08:15:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/data/data
08:15:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:15:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:15:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:15:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:15:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:15:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:15:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:15:51.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:15:51.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:15:51.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:15:51.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:15:51.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.808020 4121 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:15:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.809739 4121 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:15:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.810289 4121 kserver.cc:163] Server-wide thread pool size limit: 3276
08:15:51.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.880398 4121 rpc_server.cc:307] RPC server started. Bound to: 127.4.3.126:33713
08:15:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.880467 4188 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.3.126:33713 every 8 connection(s)
08:15:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.883391 4121 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/data/info.pb
08:15:51.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.886889 4109 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 4121
08:15:51.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.887475 4109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-0/wal/instance
08:15:51.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.890182 4189 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.892855 4109 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/wal
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/data
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:51.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/logs
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/data/info.pb
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:15:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.125:41735
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.125
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:15:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37137/jwks.json with env {}
08:15:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.906759 4189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } has no permanent_uuid. Determining permanent_uuid...
08:15:51.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.933396 4189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } has no permanent_uuid. Determining permanent_uuid...
08:15:51.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:51.935873 4138 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.3.125:41735: connect: Connection refused (error 111)
08:15:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:51.940274 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.125:41735: Network error: Client connection negotiation failed: client connection to 127.4.3.125:41735: connect: Connection refused (error 111)
08:15:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:51.989888 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } attempt: 1
08:15:51.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:51.994572 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.125:41735: Network error: Client connection negotiation failed: client connection to 127.4.3.125:41735: connect: Connection refused (error 111)
08:15:52.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:52.063123 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } attempt: 2
08:15:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.067728 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.125:41735: Network error: Client connection negotiation failed: client connection to 127.4.3.125:41735: connect: Connection refused (error 111)
08:15:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:52.159215 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } attempt: 3
08:15:52.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.163297 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.125:41735: Network error: Client connection negotiation failed: client connection to 127.4.3.125:41735: connect: Connection refused (error 111)
08:15:52.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.213061 4191 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:15:52.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.213708 4191 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:15:52.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.214249 4191 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:15:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.245003 4191 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:15:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.245322 4191 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:15:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.245584 4191 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:15:52.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.245834 4191 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:15:52.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.246031 4191 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:15:52.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.246169 4191 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37137/jwks.json
08:15:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:52.280757 4191 master_runner.cc:386] Master server non-default flags:
08:15:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/data
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/wal
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.125:41735
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:15:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37137/jwks.json
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/data/info.pb
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.125
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/logs
08:15:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:15:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:15:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:15:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:15:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:15:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:15:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:15:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:15:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:15:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:52.282019 4191 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:15:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:52.283722 4191 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:15:52.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.300010 4200 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:52.306911 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } attempt: 4
08:15:52.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.310950 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.125:41735: Network error: Client connection negotiation failed: client connection to 127.4.3.125:41735: connect: Connection refused (error 111)
08:15:52.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:52.610360 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } attempt: 5
08:15:52.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.615828 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.125:41735: Network error: Client connection negotiation failed: client connection to 127.4.3.125:41735: connect: Connection refused (error 111)
08:15:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.163245 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } attempt: 6
08:15:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.174687 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.125:41735: Network error: Client connection negotiation failed: client connection to 127.4.3.125:41735: connect: Connection refused (error 111)
08:15:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.300444 4203 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:52.303999 4191 server_base.cc:1034] running on GCE node
08:15:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:52.302160 4201 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.414484 4191 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:15:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.417263 4191 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:15:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.418687 4191 hybrid_clock.cc:648] HybridClock initialized: now 1714551353418644 us; error 47 us; skew 500 ppm
08:15:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.419756 4191 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:15:53.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.426127 4191 webserver.cc:460] Webserver started at http://127.4.3.125:39901/ using document root <none> and password file <none>
08:15:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.427075 4191 fs_manager.cc:359] Metadata directory not provided
08:15:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.427289 4191 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:15:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.427745 4191 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:15:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.432757 4191 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/data/instance:
08:15:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d2d157635924edb8023cc274be139fa"
08:15:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:53 on dist-test-slave-60sj"
08:15:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.433941 4191 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/wal/instance:
08:15:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d2d157635924edb8023cc274be139fa"
08:15:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:53 on dist-test-slave-60sj"
08:15:53.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.441231 4191 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
08:15:53.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.446848 4212 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:15:53.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.448078 4191 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:15:53.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.448395 4191 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/wal
08:15:53.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d2d157635924edb8023cc274be139fa"
08:15:53.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:53 on dist-test-slave-60sj"
08:15:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.448719 4191 fs_report.cc:383] FS layout report
08:15:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:15:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/wal
08:15:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/wal
08:15:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/data/data
08:15:53.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:15:53.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:15:53.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:15:53.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:15:53.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:15:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:15:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:15:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:15:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:15:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:15:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:15:53.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.520236 4191 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:15:53.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.521719 4191 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:15:53.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.522136 4191 kserver.cc:163] Server-wide thread pool size limit: 3276
08:15:53.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.591154 4191 rpc_server.cc:307] RPC server started. Bound to: 127.4.3.125:41735
08:15:53.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.591271 4263 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.3.125:41735 every 8 connection(s)
08:15:53.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.593828 4191 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/data/info.pb
08:15:53.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.599159 4264 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:15:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.601188 4109 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 4191
08:15:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.601574 4109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-1/wal/instance
08:15:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.603606 4109 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/wal
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/data
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/logs
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/data/info.pb
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:15:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.124:43975
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.124
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:15:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:15:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:15:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37137/jwks.json with env {}
08:15:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.612314 4264 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } has no permanent_uuid. Determining permanent_uuid...
08:15:53.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.638453 4264 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } has no permanent_uuid. Determining permanent_uuid...
08:15:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.653590 4264 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } has no permanent_uuid. Determining permanent_uuid...
08:15:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.656194 4216 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:53.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.660446 4264 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.709950 4264 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 1
08:15:53.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.714072 4264 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:53.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.782433 4264 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 2
08:15:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.787096 4264 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.878623 4264 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 3
08:15:53.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.883512 4264 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:53.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.920809 4266 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:15:53.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.921530 4266 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:15:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.922180 4266 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:15:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.952953 4266 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:15:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.953330 4266 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:15:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.953683 4266 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:15:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.954031 4266 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:15:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.954423 4266 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:15:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:53.954661 4266 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37137/jwks.json
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.989713 4266 master_runner.cc:386] Master server non-default flags:
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/data
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/wal
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.124:43975
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37137/jwks.json
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/data/info.pb
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.124
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:15:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/logs
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:15:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.991255 4266 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:15:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:53.993431 4266 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:15:54.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.009594 4279 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:54.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.027031 4264 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 4
08:15:54.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.031210 4264 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:54.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.235386 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } attempt: 7
08:15:54.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.248132 4189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } has no permanent_uuid. Determining permanent_uuid...
08:15:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.253623 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.312323 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 1
08:15:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.316679 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.330837 4264 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 5
08:15:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.336869 4264 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:54.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.398350 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 2
08:15:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.402702 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:54.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.488319 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 3
08:15:54.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.492538 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:54.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.633183 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 4
08:15:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.637120 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.884276 4264 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 6
08:15:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.891451 4264 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.920557 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 5
08:15:54.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.924710 4189 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.3.124:43975: Network error: Client connection negotiation failed: client connection to 127.4.3.124:43975: connect: Connection refused (error 111)
08:15:55.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.011068 4277 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:55.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:54.012120 4276 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:55.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:54.013723 4266 server_base.cc:1034] running on GCE node
08:15:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.136555 4266 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:15:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.139175 4266 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:15:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.140550 4266 hybrid_clock.cc:648] HybridClock initialized: now 1714551355140527 us; error 50 us; skew 500 ppm
08:15:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.141353 4266 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:15:55.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.155627 4266 webserver.cc:460] Webserver started at http://127.4.3.124:34807/ using document root <none> and password file <none>
08:15:55.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.156553 4266 fs_manager.cc:359] Metadata directory not provided
08:15:55.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.156745 4266 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:15:55.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.157128 4266 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:15:55.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.162094 4266 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/data/instance:
08:15:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9"
08:15:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:55 on dist-test-slave-60sj"
08:15:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.163283 4266 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/wal/instance:
08:15:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9"
08:15:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:55 on dist-test-slave-60sj"
08:15:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.173071 4266 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
08:15:55.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.180614 4291 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:15:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.181891 4266 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
08:15:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.182216 4266 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/wal
08:15:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9"
08:15:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:55 on dist-test-slave-60sj"
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.182642 4266 fs_report.cc:383] FS layout report
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/wal
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/wal
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/data/data
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:15:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:15:55.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.232759 4266 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:15:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.234333 4266 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:15:55.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.234773 4266 kserver.cc:163] Server-wide thread pool size limit: 3276
08:15:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.304430 4266 rpc_server.cc:307] RPC server started. Bound to: 127.4.3.124:43975
08:15:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.304522 4342 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.3.124:43975 every 8 connection(s)
08:15:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.307579 4266 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/data/info.pb
08:15:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.309785 4109 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 4266
08:15:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.310282 4109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/master-2/wal/instance
08:15:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.314024 4343 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:15:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.332207 4343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } has no permanent_uuid. Determining permanent_uuid...
08:15:55.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.356444 4343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } has no permanent_uuid. Determining permanent_uuid...
08:15:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.366268 4343 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } has no permanent_uuid. Determining permanent_uuid...
08:15:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.380352 4343 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9: Bootstrap starting.
08:15:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.387282 4343 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9: Neither blocks nor log segments found. Creating new log.
08:15:55.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.389720 4343 log.cc:824] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9: Log is configured to *not* fsync() on all Append() calls
08:15:55.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.394493 4343 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9: No bootstrap required, opened a new log
08:15:55.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.416105 4343 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.416829 4343 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:15:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.417028 4343 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7cdbe2e7068c4e2785eaf9de4ba251d9, State: Initialized, Role: FOLLOWER
08:15:55.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.417760 4343 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.420341 4352 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.421535 4352 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: This master's current role is: FOLLOWER
08:15:55.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.422302 4343 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: configured and running, proceeding with master startup.
08:15:55.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:55.447988 4363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:15:55.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:55.448382 4363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:15:55.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.442798 4266 master_runner.cc:186] Error getting master registration for 127.4.3.126:33713: OK, instance_id { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" instance_seqno: 1714551351860971 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:15:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.465704 4266 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:15:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.477128 4189 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 6
08:15:55.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:55.490139 4297 tablet.cc:2369] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:15:55.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.495708 4189 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: Bootstrap starting.
08:15:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.501041 4189 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: Neither blocks nor log segments found. Creating new log.
08:15:55.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.503182 4189 log.cc:824] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: Log is configured to *not* fsync() on all Append() calls
08:15:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.507807 4189 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: No bootstrap required, opened a new log
08:15:55.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.530880 4189 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:55.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.531561 4189 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:15:55.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.531883 4189 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1a46e5d93bba4cc3b46fa40f0c7b36a7, State: Initialized, Role: FOLLOWER
08:15:55.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.533187 4189 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.536526 4367 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.537456 4367 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [sys.catalog]: This master's current role is: FOLLOWER
08:15:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.538326 4189 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [sys.catalog]: configured and running, proceeding with master startup.
08:15:55.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:55.569265 4143 tablet.cc:2369] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:15:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:55.570814 4378 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:15:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:55.571465 4378 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:15:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.587632 4352 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:15:55.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.588215 4352 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.592500 4164 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" is_pre_election: true
08:15:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.593451 4164 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7cdbe2e7068c4e2785eaf9de4ba251d9 in term 0.
08:15:55.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.595464 4293 leader_election.cc:302] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a46e5d93bba4cc3b46fa40f0c7b36a7, 7cdbe2e7068c4e2785eaf9de4ba251d9; no voters:
08:15:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.599246 4352 leader_election.cc:288] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1a46e5d93bba4cc3b46fa40f0c7b36a7 (127.4.3.126:33713), 3d2d157635924edb8023cc274be139fa (127.4.3.125:41735)
08:15:55.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.601025 4352 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:15:55.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.601893 4352 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:15:55.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.602726 4352 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 0 FOLLOWER]: Advancing to term 1
08:15:55.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.626106 4352 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:55.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.571378 4121 master_runner.cc:186] Error getting master registration for 127.4.3.125:41735: OK, instance_id { permanent_uuid: "3d2d157635924edb8023cc274be139fa" instance_seqno: 1714551353571750 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:15:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.638949 4164 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7"
08:15:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.642253 4121 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:15:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.642306 4164 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [term 0 FOLLOWER]: Advancing to term 1
08:15:55.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.641580 4352 leader_election.cc:288] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [CANDIDATE]: Term 1 election: Requested vote from peers 1a46e5d93bba4cc3b46fa40f0c7b36a7 (127.4.3.126:33713), 3d2d157635924edb8023cc274be139fa (127.4.3.125:41735)
08:15:55.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.657384 4164 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7cdbe2e7068c4e2785eaf9de4ba251d9 in term 1.
08:15:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.663431 4293 leader_election.cc:302] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1a46e5d93bba4cc3b46fa40f0c7b36a7, 7cdbe2e7068c4e2785eaf9de4ba251d9; no voters:
08:15:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.665258 4352 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 1 FOLLOWER]: Leader election won for term 1
08:15:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.666620 4352 raft_consensus.cc:686] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [term 1 LEADER]: Becoming Leader. State: Replica: 7cdbe2e7068c4e2785eaf9de4ba251d9, State: Running, Role: LEADER
08:15:55.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.669827 4352 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.681723 4380 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cdbe2e7068c4e2785eaf9de4ba251d9. Latest consensus state: current_term: 1 leader_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.683023 4380 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: This master's current role is: LEADER
08:15:55.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.696310 4382 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
08:15:55.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.704452 4382 catalog_manager.cc:1484] Initializing Kudu cluster ID...
08:15:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.737278 4164 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [term 1 FOLLOWER]: Refusing update from remote peer 7cdbe2e7068c4e2785eaf9de4ba251d9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:15:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.741649 4352 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
08:15:55.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.774645 4367 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cdbe2e7068c4e2785eaf9de4ba251d9. Latest consensus state: current_term: 1 leader_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:55.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.775434 4367 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [sys.catalog]: This master's current role is: FOLLOWER
08:15:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.779847 4383 mvcc.cc:205] Tried to move back new op lower bound from 7022802353032171520 to 7022802352846700544. Current Snapshot: MvccSnapshot[applied={T|T < 7022802353032171520}]
08:15:55.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.785950 4384 mvcc.cc:205] Tried to move back new op lower bound from 7022802353032171520 to 7022802352846700544. Current Snapshot: MvccSnapshot[applied={T|T < 7022802353032171520}]
08:15:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.804694 4382 catalog_manager.cc:1347] Generated new cluster ID: 39ba231b9e7e499eb476ebe07d943895
08:15:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.805022 4382 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
08:15:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.811038 4367 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:55.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.812157 4367 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7 [sys.catalog]: This master's current role is: FOLLOWER
08:15:55.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.815430 4352 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:55.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.816248 4352 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: This master's current role is: LEADER
08:15:55.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.855832 4382 catalog_manager.cc:1370] Generated new certificate authority record
08:15:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.860350 4382 catalog_manager.cc:1504] Loading token signing keys...
08:15:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.897688 4382 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9: Generated new TSK 0
08:15:55.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.899966 4382 catalog_manager.cc:1514] Initializing in-progress tserver states...
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.929702 4109 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/wal
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/data
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/logs
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/data/info.pb
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:15:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.65:0
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.3.65
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.65
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:15:55.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:55.951823 4264 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } attempt: 7
08:15:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.002244 4264 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa: Bootstrap starting.
08:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.021450 4264 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa: Neither blocks nor log segments found. Creating new log.
08:15:56.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.027805 4264 log.cc:824] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa: Log is configured to *not* fsync() on all Append() calls
08:15:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.044569 4264 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa: No bootstrap required, opened a new log
08:15:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.132009 4264 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:56.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.134524 4264 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:15:56.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.135298 4264 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d2d157635924edb8023cc274be139fa, State: Initialized, Role: FOLLOWER
08:15:56.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.138213 4264 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } }
08:15:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.160940 4238 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d2d157635924edb8023cc274be139fa"
08:15:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.161808 4238 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [term 0 FOLLOWER]: Advancing to term 1
08:15:56.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.163303 4239 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d2d157635924edb8023cc274be139fa" is_pre_election: true
08:15:56.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.173251 4238 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7cdbe2e7068c4e2785eaf9de4ba251d9 in term 1.
08:15:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.175083 4237 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [term 1 FOLLOWER]: Refusing update from remote peer 7cdbe2e7068c4e2785eaf9de4ba251d9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:15:56.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.182344 4380 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:15:56.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.185554 4264 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [sys.catalog]: configured and running, proceeding with master startup.
08:15:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.183578 4391 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.187572 4391 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [sys.catalog]: This master's current role is: FOLLOWER
08:15:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.262668 4391 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cdbe2e7068c4e2785eaf9de4ba251d9. Latest consensus state: current_term: 1 leader_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.263460 4391 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [sys.catalog]: This master's current role is: FOLLOWER
08:15:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.264662 4399 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.265699 4399 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa [sys.catalog]: This master's current role is: FOLLOWER
08:15:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.299351 4407 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:15:56.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:56.468631 4388 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:15:56.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:56.469295 4388 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:15:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:56.470099 4388 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:15:56.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:56.526367 4388 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:15:56.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:56.527822 4388 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.3.65
08:15:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.577658 4378 catalog_manager.cc:1259] Loaded cluster ID: 39ba231b9e7e499eb476ebe07d943895
08:15:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.578403 4378 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: loading cluster ID for follower catalog manager: success
08:15:56.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.587802 4378 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: acquiring CA information for follower catalog manager: success
08:15:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.590366 4388 tablet_server_runner.cc:78] Tablet server non-default flags:
08:15:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/data
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/wal
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.65:0
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:56.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/data/info.pb
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.65
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.3.65
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/logs
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:15:56.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:15:56.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:15:56.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:15:56.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:15:56.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.591898 4388 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:15:56.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.594044 4388 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:15:56.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.618960 4378 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1a46e5d93bba4cc3b46fa40f0c7b36a7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:15:56.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:56.650414 4414 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:56.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.723356 4380 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1a46e5d93bba4cc3b46fa40f0c7b36a7" member_type: VOTER last_known_addr { host: "127.4.3.126" port: 33713 } } peers { permanent_uuid: "3d2d157635924edb8023cc274be139fa" member_type: VOTER last_known_addr { host: "127.4.3.125" port: 41735 } } peers { permanent_uuid: "7cdbe2e7068c4e2785eaf9de4ba251d9" member_type: VOTER last_known_addr { host: "127.4.3.124" port: 43975 } } }
08:15:56.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:56.724792 4380 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cdbe2e7068c4e2785eaf9de4ba251d9 [sys.catalog]: This master's current role is: LEADER
08:15:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:57.329149 4407 catalog_manager.cc:1259] Loaded cluster ID: 39ba231b9e7e499eb476ebe07d943895
08:15:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:57.334394 4407 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa: loading cluster ID for follower catalog manager: success
08:15:57.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:57.358955 4407 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa: acquiring CA information for follower catalog manager: success
08:15:57.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:57.365132 4407 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3d2d157635924edb8023cc274be139fa: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:15:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:58.000048 4415 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:15:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.000252 4388 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:15:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:56.664791 4417 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:58.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:56.654572 4413 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:58.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.006892 4388 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:15:58.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.018659 4388 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:15:58.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.020136 4388 hybrid_clock.cc:648] HybridClock initialized: now 1714551358020100 us; error 50 us; skew 500 ppm
08:15:58.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.021234 4388 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:15:58.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.058250 4388 webserver.cc:460] Webserver started at http://127.4.3.65:40513/ using document root <none> and password file <none>
08:15:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.059592 4388 fs_manager.cc:359] Metadata directory not provided
08:15:58.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.059886 4388 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:15:58.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.060468 4388 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:15:58.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.068037 4388 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/data/instance:
08:15:58.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8d027af3793423e9f5098d123260877"
08:15:58.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:58 on dist-test-slave-60sj"
08:15:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.069669 4388 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/wal/instance:
08:15:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8d027af3793423e9f5098d123260877"
08:15:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:58 on dist-test-slave-60sj"
08:15:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.086247 4388 fs_manager.cc:691] Time spent creating directory manager: real 0.016s user 0.005s sys 0.008s
08:15:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.097771 4426 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:15:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.099261 4388 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.002s sys 0.005s
08:15:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.099687 4388 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/wal
08:15:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e8d027af3793423e9f5098d123260877"
08:15:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:15:58 on dist-test-slave-60sj"
08:15:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.100148 4388 fs_report.cc:383] FS layout report
08:15:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:15:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/wal
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/wal
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/data/data
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:15:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:15:58.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.187717 4388 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:15:58.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.189796 4388 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:15:58.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.190421 4388 kserver.cc:163] Server-wide thread pool size limit: 3276
08:15:58.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.194447 4388 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:15:58.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.200536 4388 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:15:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.200832 4388 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:15:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.201157 4388 ts_tablet_manager.cc:590] Registered 0 tablets
08:15:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.201370 4388 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:15:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.449127 4388 rpc_server.cc:307] RPC server started. Bound to: 127.4.3.65:43311
08:15:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.449256 4540 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.3.65:43311 every 8 connection(s)
08:15:58.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.452769 4388 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/data/info.pb
08:15:58.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.461076 4109 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 4388
08:15:58.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.461877 4109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-0/wal/instance
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.484838 4109 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/wal
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/data
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/logs
08:15:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/data/info.pb
08:15:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:15:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:15:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:15:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:15:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.66:0
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.3.66
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.66
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:58.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:15:58.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.565115 4541 heartbeater.cc:347] Connected to a master server at 127.4.3.124:43975
08:15:58.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.565660 4541 heartbeater.cc:464] Registering TS with master...
08:15:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.566958 4541 heartbeater.cc:511] Master 127.4.3.124:43975 requested a full tablet report, sending...
08:15:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.570235 4308 ts_manager.cc:194] Registered new tserver with Master: e8d027af3793423e9f5098d123260877 (127.4.3.65:43311)
08:15:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.573135 4308 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.3.65:44357
08:15:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.581107 4542 heartbeater.cc:347] Connected to a master server at 127.4.3.125:41735
08:15:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.581521 4542 heartbeater.cc:464] Registering TS with master...
08:15:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.582193 4542 heartbeater.cc:511] Master 127.4.3.125:41735 requested a full tablet report, sending...
08:15:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.586072 4229 ts_manager.cc:194] Registered new tserver with Master: e8d027af3793423e9f5098d123260877 (127.4.3.65:43311)
08:15:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.594760 4546 heartbeater.cc:347] Connected to a master server at 127.4.3.126:33713
08:15:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.595153 4546 heartbeater.cc:464] Registering TS with master...
08:15:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.595814 4546 heartbeater.cc:511] Master 127.4.3.126:33713 requested a full tablet report, sending...
08:15:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:58.599428 4154 ts_manager.cc:194] Registered new tserver with Master: e8d027af3793423e9f5098d123260877 (127.4.3.65:43311)
08:15:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:58.958542 4545 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:15:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:58.959018 4545 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:15:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:58.959481 4545 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:15:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:58.991057 4545 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:15:58.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:58.991868 4545 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.3.66
08:15:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:59.026468 4545 tablet_server_runner.cc:78] Tablet server non-default flags:
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/data
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/wal
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.66:0
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/data/info.pb
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.66
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:15:59.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.3.66
08:15:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/logs
08:15:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:15:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:15:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:15:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:15:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:15:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:15:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:15:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:15:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:15:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:15:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:59.027617 4545 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:15:59.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:59.029505 4545 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:15:59.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:59.054479 4557 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:15:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:15:59.577992 4541 heartbeater.cc:503] Master 127.4.3.124:43975 was elected leader, sending a full tablet report...
08:16:00.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:59.062153 4560 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:00.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:00.307121 4559 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1253 milliseconds
08:16:00.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.307143 4545 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:16:00.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:15:59.055390 4558 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:00.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.312139 4545 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:00.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.316090 4545 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:00.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.317724 4545 hybrid_clock.cc:648] HybridClock initialized: now 1714551360317645 us; error 83 us; skew 500 ppm
08:16:00.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.319010 4545 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.333695 4545 webserver.cc:460] Webserver started at http://127.4.3.66:46807/ using document root <none> and password file <none>
08:16:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.334611 4545 fs_manager.cc:359] Metadata directory not provided
08:16:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.334820 4545 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:00.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.335222 4545 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.340517 4545 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/data/instance:
08:16:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8d1e25cc5c649cfb64137b517a160db"
08:16:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:00 on dist-test-slave-60sj"
08:16:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.341681 4545 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/wal/instance:
08:16:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8d1e25cc5c649cfb64137b517a160db"
08:16:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:00 on dist-test-slave-60sj"
08:16:00.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.349408 4545 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
08:16:00.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.355475 4568 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.356585 4545 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.356865 4545 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/wal
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c8d1e25cc5c649cfb64137b517a160db"
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:00 on dist-test-slave-60sj"
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.357182 4545 fs_report.cc:383] FS layout report
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/wal
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/wal
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/data/data
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:00.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:00.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:00.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:00.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:00.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:00.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:00.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.417246 4545 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:00.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.418747 4545 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:00.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.419162 4545 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:00.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.422111 4545 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:16:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.426352 4545 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:16:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.426548 4545 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:16:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.426739 4545 ts_tablet_manager.cc:590] Registered 0 tablets
08:16:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.426880 4545 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:16:00.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.583647 4545 rpc_server.cc:307] RPC server started. Bound to: 127.4.3.66:35897
08:16:00.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.583756 4681 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.3.66:35897 every 8 connection(s)
08:16:00.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.586071 4545 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/data/info.pb
08:16:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.586908 4109 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 4545
08:16:00.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.587486 4109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-1/wal/instance
08:16:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.601269 4109 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/wal
08:16:00.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/data
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/logs
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/data/info.pb
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.67:0
08:16:00.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.3.67
08:16:00.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.67
08:16:00.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:00.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:16:00.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:16:00.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:00.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:00.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:16:00.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.658957 4683 heartbeater.cc:347] Connected to a master server at 127.4.3.125:41735
08:16:00.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.659473 4683 heartbeater.cc:464] Registering TS with master...
08:16:00.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.660790 4683 heartbeater.cc:511] Master 127.4.3.125:41735 requested a full tablet report, sending...
08:16:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.663576 4684 heartbeater.cc:347] Connected to a master server at 127.4.3.126:33713
08:16:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.663913 4684 heartbeater.cc:464] Registering TS with master...
08:16:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.664489 4229 ts_manager.cc:194] Registered new tserver with Master: c8d1e25cc5c649cfb64137b517a160db (127.4.3.66:35897)
08:16:00.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.664801 4684 heartbeater.cc:511] Master 127.4.3.126:33713 requested a full tablet report, sending...
08:16:00.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.665710 4682 heartbeater.cc:347] Connected to a master server at 127.4.3.124:43975
08:16:00.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.666152 4682 heartbeater.cc:464] Registering TS with master...
08:16:00.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.667021 4682 heartbeater.cc:511] Master 127.4.3.124:43975 requested a full tablet report, sending...
08:16:00.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.667912 4154 ts_manager.cc:194] Registered new tserver with Master: c8d1e25cc5c649cfb64137b517a160db (127.4.3.66:35897)
08:16:00.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.669080 4308 ts_manager.cc:194] Registered new tserver with Master: c8d1e25cc5c649cfb64137b517a160db (127.4.3.66:35897)
08:16:00.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:00.670526 4308 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.3.66:33957
08:16:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:00.968926 4691 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:00.969412 4691 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:00.969903 4691 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:01.000202 4691 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:01.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:01.000993 4691 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.3.67
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:01.034816 4691 tablet_server_runner.cc:78] Tablet server non-default flags:
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.3.84:35607
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/data
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/wal
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.3.67:0
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/data/info.pb
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.3.67
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.3.126:33713,127.4.3.125:41735,127.4.3.124:43975
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:01.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.3.67
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/logs
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:01.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:01.036109 4691 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:01.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:01.037925 4691 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:01.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:01.059221 4698 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:01.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:01.674006 4682 heartbeater.cc:503] Master 127.4.3.124:43975 was elected leader, sending a full tablet report...
08:16:01.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:01.731336 4185 debug-util.cc:398] Leaking SignalData structure 0x7b0800073b80 after lost signal to thread 4122
08:16:01.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:01.732460 4185 debug-util.cc:398] Leaking SignalData structure 0x7b08000c2020 after lost signal to thread 4188
08:16:02.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:01.062853 4691 server_base.cc:1034] running on GCE node
08:16:02.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:01.061465 4701 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:02.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:01.059672 4699 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:02.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.242089 4691 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:02.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.244885 4691 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:02.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.246588 4691 hybrid_clock.cc:648] HybridClock initialized: now 1714551362246523 us; error 100 us; skew 500 ppm
08:16:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.247538 4691 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.255056 4691 webserver.cc:460] Webserver started at http://127.4.3.67:41675/ using document root <none> and password file <none>
08:16:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.256175 4691 fs_manager.cc:359] Metadata directory not provided
08:16:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.256403 4691 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.256953 4691 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:02.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.262355 4691 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/data/instance:
08:16:02.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "829f939d5bdf4263badbda62699e5d7c"
08:16:02.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:02 on dist-test-slave-60sj"
08:16:02.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.263535 4691 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/wal/instance:
08:16:02.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "829f939d5bdf4263badbda62699e5d7c"
08:16:02.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:02 on dist-test-slave-60sj"
08:16:02.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.271908 4691 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
08:16:02.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.278432 4709 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:02.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.279538 4691 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.279835 4691 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/wal
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "829f939d5bdf4263badbda62699e5d7c"
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:02 on dist-test-slave-60sj"
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.280158 4691 fs_report.cc:383] FS layout report
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/wal
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/wal
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/data/data
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:02.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.333716 4691 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:02.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.335283 4691 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:02.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.335745 4691 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:02.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.338990 4691 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:16:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.344427 4691 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:16:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.344728 4691 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:16:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.345046 4691 ts_tablet_manager.cc:590] Registered 0 tablets
08:16:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.345233 4691 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:16:02.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.549425 4691 rpc_server.cc:307] RPC server started. Bound to: 127.4.3.67:38349
08:16:02.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.549604 4821 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.3.67:38349 every 8 connection(s)
08:16:02.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.552886 4691 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/data/info.pb
08:16:02.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.561282 4109 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 4691
08:16:02.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.561786 4109 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster6160923401061251491/ts-2/wal/instance
08:16:02.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.626307 4822 heartbeater.cc:347] Connected to a master server at 127.4.3.124:43975
08:16:02.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.626822 4822 heartbeater.cc:464] Registering TS with master...
08:16:02.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.628180 4822 heartbeater.cc:511] Master 127.4.3.124:43975 requested a full tablet report, sending...
08:16:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.631618 4307 ts_manager.cc:194] Registered new tserver with Master: 829f939d5bdf4263badbda62699e5d7c (127.4.3.67:38349)
08:16:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.631647 4824 heartbeater.cc:347] Connected to a master server at 127.4.3.126:33713
08:16:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.632817 4824 heartbeater.cc:464] Registering TS with master...
08:16:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.633407 4307 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.3.67:53799
08:16:02.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.637984 4824 heartbeater.cc:511] Master 127.4.3.126:33713 requested a full tablet report, sending...
08:16:02.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.640841 4154 ts_manager.cc:194] Registered new tserver with Master: 829f939d5bdf4263badbda62699e5d7c (127.4.3.67:38349)
08:16:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.643196 4823 heartbeater.cc:347] Connected to a master server at 127.4.3.125:41735
08:16:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.643564 4823 heartbeater.cc:464] Registering TS with master...
08:16:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.644228 4823 heartbeater.cc:511] Master 127.4.3.125:41735 requested a full tablet report, sending...
08:16:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.647423 4228 ts_manager.cc:194] Registered new tserver with Master: 829f939d5bdf4263badbda62699e5d7c (127.4.3.67:38349)
08:16:02.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:02.659040 4109 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
08:16:02.670 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:16:02.671 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:16:02.689 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.3.126"
port: 33713
}
bound_http_address {
host: "127.4.3.126"
port: 41663
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.3.125"
port: 41735
}
bound_http_address {
host: "127.4.3.125"
port: 39901
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.3.124"
port: 43975
}
bound_http_address {
host: "127.4.3.124"
port: 34807
}
}
}
08:16:02.693 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:16:02.696 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.3.65"
port: 43311
}
bound_http_address {
host: "127.4.3.65"
port: 40513
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.3.66"
port: 35897
}
bound_http_address {
host: "127.4.3.66"
port: 46807
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.3.67"
port: 38349
}
bound_http_address {
host: "127.4.3.67"
port: 41675
}
}
}
08:16:02.696 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:16:02.919 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:16:03.050 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0YutnF/test-tmp
08:16:03.051 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:16:03.051 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0YutnF/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:16:03.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:16:03.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:03.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:16:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:03.434695 4833 mini_oidc.cc:152] Starting JWKS server
08:16:03.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:03.438843 4833 webserver.cc:460] Webserver started at http://127.0.0.1:44881/ using document root <none> and password file <none>
08:16:03.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:03.439327 4833 mini_oidc.cc:189] Starting OIDC Discovery server
08:16:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:03.441577 4833 webserver.cc:460] Webserver started at http://127.0.0.1:41439/ using document root <none> and password file <none>
08:16:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:16:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/principal' for realm 'KRBTEST.COM',
08:16:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:16:03.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:03 dist-test-slave-60sj krb5kdc[4845](info): setting up network...
08:16:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:16:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:03 dist-test-slave-60sj krb5kdc[4845](info): set up 2 sockets
08:16:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:03 dist-test-slave-60sj krb5kdc[4845](info): commencing operation
08:16:03.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:16:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:03.638309 4822 heartbeater.cc:503] Master 127.4.3.124:43975 was elected leader, sending a full tablet report...
08:16:03.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:03.825238 4260 debug-util.cc:398] Leaking SignalData structure 0x7b080008f5c0 after lost signal to thread 4195
08:16:03.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:03.825903 4260 debug-util.cc:398] Leaking SignalData structure 0x7b080009ae40 after lost signal to thread 4263
08:16:04.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:04.577637 4833 mini_kdc.cc:121] Time spent starting KDC: real 1.090s user 0.010s sys 0.014s
08:16:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:16:04.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:16:04.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:16:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:16:04.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:16:04.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:16:04.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:16:04.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:16:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:16:04.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:16:04.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/test-user.keytab.
08:16:04.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/test-user.keytab.
08:16:04.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:04 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:04.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:16:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:16:04Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:16:04.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:16:04Z Disabled control of system clock
08:16:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.184.126@KRBTEST.COM; defaulting to no policy
08:16:04.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:04.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.184.126@KRBTEST.COM" created.
08:16:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.184.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.126.keytab.
08:16:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.184.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.126.keytab.
08:16:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:04.802297 4833 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/logs
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/info.pb
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.126:45717
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.126
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.126.keytab
08:16:04.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.126
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:05.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.132862 4861 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:05.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.133555 4861 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:05.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.134274 4861 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:05.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.165539 4861 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:05.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.166082 4861 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:05.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.166899 4861 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:05.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.167428 4861 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.168051 4861 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.169035 4861 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.169633 4861 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44881/jwks.json
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:05.207648 4861 master_runner.cc:386] Master server non-default flags:
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.126.keytab
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.126
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.126:45717
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/info.pb
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.126
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/logs
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:16:05.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:16:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:16:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:05.208776 4861 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:05.209810 4861 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:05.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.233189 4867 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.235702 4868 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:05.238045 4870 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:05.239728 4861 server_base.cc:1034] running on GCE node
08:16:06.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.412521 4861 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.416325 4861 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:06.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.417819 4861 hybrid_clock.cc:648] HybridClock initialized: now 1714551366417736 us; error 80 us; skew 500 ppm
08:16:06.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.419098 4861 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:06 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551366, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.431288 4861 init.cc:377] Logged in from keytab as kudu/127.4.184.126@KRBTEST.COM (short username kudu)
08:16:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.436587 4861 webserver.cc:460] Webserver started at http://127.4.184.126:35137/ using document root <none> and password file <none>
08:16:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.437575 4861 fs_manager.cc:359] Metadata directory not provided
08:16:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.437798 4861 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:06.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.438292 4861 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:06.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.443318 4861 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/instance:
08:16:06.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ac666bfd8b5447282e0bf43f7de44e9"
08:16:06.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:06 on dist-test-slave-60sj"
08:16:06.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.444583 4861 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal/instance:
08:16:06.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ac666bfd8b5447282e0bf43f7de44e9"
08:16:06.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:06 on dist-test-slave-60sj"
08:16:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.452735 4861 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.005s
08:16:06.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.458608 4879 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.459661 4861 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:16:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.460011 4861 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ac666bfd8b5447282e0bf43f7de44e9"
08:16:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:06 on dist-test-slave-60sj"
08:16:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.460399 4861 fs_report.cc:383] FS layout report
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/data
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:06.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.510947 4861 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.516860 4861 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.517377 4861 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:06.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.604488 4861 rpc_server.cc:307] RPC server started. Bound to: 127.4.184.126:45717
08:16:06.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.604625 4930 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.184.126:45717 every 8 connection(s)
08:16:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.607681 4861 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/info.pb
08:16:06.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.609726 4833 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 4861
08:16:06.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.610213 4833 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal/instance
08:16:06.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.614264 4931 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.184.125@KRBTEST.COM; defaulting to no policy
08:16:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.626935 4931 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } has no permanent_uuid. Determining permanent_uuid...
08:16:06.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:06.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.184.125@KRBTEST.COM" created.
08:16:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:06 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551366, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.126@KRBTEST.COM for kudu/127.4.184.126@KRBTEST.COM
08:16:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.184.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.125.keytab.
08:16:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.184.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.125.keytab.
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.662860 4833 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/logs
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/info.pb
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.125:38831
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.125
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.125.keytab
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.125
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:06.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.674340 4931 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } has no permanent_uuid. Determining permanent_uuid...
08:16:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:06.677376 4881 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:06.682709 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.125:38831: Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.732297 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } attempt: 1
08:16:06.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:06.736503 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.125:38831: Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.804879 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } attempt: 2
08:16:06.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:06.809463 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.125:38831: Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:06.900871 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } attempt: 3
08:16:06.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:06.904706 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.125:38831: Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:06.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:06.985697 4937 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:06.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:06.986238 4937 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:06.986934 4937 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.017930 4937 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.018177 4937 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.018435 4937 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.018633 4937 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.018822 4937 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.019009 4937 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.019160 4937 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44881/jwks.json
08:16:07.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:07.048147 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } attempt: 4
08:16:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.051779 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.125:38831: Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:07.056275 4937 master_runner.cc:386] Master server non-default flags:
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.125.keytab
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.125
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.125:38831
08:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/info.pb
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.125
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/logs
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:07.057374 4937 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:07.058406 4937 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:07.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.076807 4944 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:07.351141 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } attempt: 5
08:16:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.356029 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.125:38831: Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:07.903683 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } attempt: 6
08:16:07.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.912529 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.125:38831: Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:08.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.078174 4947 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:08.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:07.079393 4937 server_base.cc:1034] running on GCE node
08:16:08.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:07.078836 4945 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:08.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.226033 4937 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:08.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.229328 4937 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.231061 4937 hybrid_clock.cc:648] HybridClock initialized: now 1714551368231029 us; error 52 us; skew 500 ppm
08:16:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.232103 4937 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:08.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:08 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551368, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.238571 4937 init.cc:377] Logged in from keytab as kudu/127.4.184.125@KRBTEST.COM (short username kudu)
08:16:08.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.243676 4937 webserver.cc:460] Webserver started at http://127.4.184.125:34243/ using document root <none> and password file <none>
08:16:08.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.244626 4937 fs_manager.cc:359] Metadata directory not provided
08:16:08.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.244920 4937 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:08.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.245395 4937 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.250763 4937 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/instance:
08:16:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a167916b70e74b248d2ab3228786e6ce"
08:16:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:08 on dist-test-slave-60sj"
08:16:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.251981 4937 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal/instance:
08:16:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a167916b70e74b248d2ab3228786e6ce"
08:16:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:08 on dist-test-slave-60sj"
08:16:08.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.259781 4937 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
08:16:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.266026 4957 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.267138 4937 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:16:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.267495 4937 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a167916b70e74b248d2ab3228786e6ce"
08:16:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:08 on dist-test-slave-60sj"
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.267854 4937 fs_report.cc:383] FS layout report
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/data
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:08.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.322844 4937 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:08.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.329246 4937 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.329823 4937 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:08.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.412042 4937 rpc_server.cc:307] RPC server started. Bound to: 127.4.184.125:38831
08:16:08.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.412123 5008 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.184.125:38831 every 8 connection(s)
08:16:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.414820 4937 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/info.pb
08:16:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.418695 4833 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 4937
08:16:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.419128 4833 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal/instance
08:16:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.421048 5009 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:08.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.184.124@KRBTEST.COM; defaulting to no policy
08:16:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.434104 5009 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } has no permanent_uuid. Determining permanent_uuid...
08:16:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.184.124@KRBTEST.COM" created.
08:16:08.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:08 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551368, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.125@KRBTEST.COM for kudu/127.4.184.126@KRBTEST.COM
08:16:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.184.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.124.keytab.
08:16:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.184.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.124.keytab.
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.465783 4833 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/logs
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/info.pb
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.124:43649
08:16:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.124
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.124.keytab
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.124
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:08.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.470868 5009 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } has no permanent_uuid. Determining permanent_uuid...
08:16:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:08 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551368, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.125@KRBTEST.COM for kudu/127.4.184.125@KRBTEST.COM
08:16:08.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.497846 5009 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } has no permanent_uuid. Determining permanent_uuid...
08:16:08.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.500006 4958 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:08.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.504380 5009 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.553887 5009 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 1
08:16:08.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.557601 5009 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:08.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.625963 5009 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 2
08:16:08.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.636854 5009 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:08.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.728307 5009 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 3
08:16:08.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.732415 5009 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:08.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.818848 5015 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:08.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.819416 5015 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.819991 5015 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:08.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.851153 5015 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:08.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.851431 5015 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:08.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.851809 5015 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:08.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.852104 5015 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:08.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.852378 5015 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:08.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.852649 5015 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:08.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.852857 5015 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44881/jwks.json
08:16:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.875793 5009 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 4
08:16:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.879835 5009 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.891088 5015 master_runner.cc:386] Master server non-default flags:
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.124.keytab
08:16:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.124
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.124:43649
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/info.pb
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.124
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/logs
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.892531 5015 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:08.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.893962 5015 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.913436 5027 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.973177 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } attempt: 7
08:16:08.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:08 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551366, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.126@KRBTEST.COM for kudu/127.4.184.125@KRBTEST.COM
08:16:08.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.994218 4931 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } has no permanent_uuid. Determining permanent_uuid...
08:16:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:09.000352 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:09.058794 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 1
08:16:09.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:09.062541 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:09.144187 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 2
08:16:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:09.148454 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:09.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:09.179258 5009 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 5
08:16:09.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:09.184602 5009 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:09.233902 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 3
08:16:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:09.239442 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:09.379904 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 4
08:16:09.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:09.385440 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:09.668860 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 5
08:16:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:09.674265 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:09.732156 5009 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 6
08:16:09.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:09.740904 5009 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.914201 5025 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:08.916693 5015 server_base.cc:1034] running on GCE node
08:16:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:08.914865 5024 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.101230 5015 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.103879 5015 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.105262 5015 hybrid_clock.cc:648] HybridClock initialized: now 1714551370105234 us; error 46 us; skew 500 ppm
08:16:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.106074 5015 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551370, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.112555 5015 init.cc:377] Logged in from keytab as kudu/127.4.184.124@KRBTEST.COM (short username kudu)
08:16:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.118981 5015 webserver.cc:460] Webserver started at http://127.4.184.124:44329/ using document root <none> and password file <none>
08:16:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.119906 5015 fs_manager.cc:359] Metadata directory not provided
08:16:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.120134 5015 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.120527 5015 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.126390 5015 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/instance:
08:16:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f"
08:16:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:10 on dist-test-slave-60sj"
08:16:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.127621 5015 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal/instance:
08:16:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f"
08:16:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:10 on dist-test-slave-60sj"
08:16:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.135411 5015 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
08:16:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.141639 5041 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.142827 5015 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:16:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.143157 5015 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f"
08:16:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:10 on dist-test-slave-60sj"
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.143484 5015 fs_report.cc:383] FS layout report
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/data
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.210268 5015 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.216220 5015 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.217176 5015 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:10.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.226686 4931 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } attempt: 6
08:16:10.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:10.234432 4931 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.4.184.124:43649: Network error: Client connection negotiation failed: client connection to 127.4.184.124:43649: connect: Connection refused (error 111)
08:16:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.300411 5015 rpc_server.cc:307] RPC server started. Bound to: 127.4.184.124:43649
08:16:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.300484 5094 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.184.124:43649 every 8 connection(s)
08:16:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.303242 5015 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/info.pb
08:16:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.303922 4833 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 5015
08:16:10.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.304459 4833 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal/instance
08:16:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.311923 5095 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.342175 5095 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } has no permanent_uuid. Determining permanent_uuid...
08:16:10.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.184.124@KRBTEST.COM
08:16:10.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551370, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.124@KRBTEST.COM for kudu/127.4.184.126@KRBTEST.COM
08:16:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.373509 5095 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } has no permanent_uuid. Determining permanent_uuid...
08:16:10.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551370, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.124@KRBTEST.COM for kudu/127.4.184.125@KRBTEST.COM
08:16:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.184.125@KRBTEST.COM
08:16:10.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.399991 5095 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } has no permanent_uuid. Determining permanent_uuid...
08:16:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551370, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.124@KRBTEST.COM for kudu/127.4.184.124@KRBTEST.COM
08:16:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.184.126@KRBTEST.COM
08:16:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.432053 5095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: Bootstrap starting.
08:16:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.432525 4833 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
08:16:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.439968 5095 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: Neither blocks nor log segments found. Creating new log.
08:16:10.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.442344 5095 log.cc:824] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: Log is configured to *not* fsync() on all Append() calls
08:16:10.443 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:10.445 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:16:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.448271 5095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: No bootstrap required, opened a new log
08:16:10.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:16:10.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:16:10.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.184.126"
port: 45717
}
bound_http_address {
host: "127.4.184.126"
port: 35137
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.184.125"
port: 38831
}
bound_http_address {
host: "127.4.184.125"
port: 34243
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.184.124"
port: 43649
}
bound_http_address {
host: "127.4.184.124"
port: 44329
}
}
}
08:16:10.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:16:10.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:10.463 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:16:10.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:16:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.475266 5095 raft_consensus.cc:348] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.475916 5095 raft_consensus.cc:374] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:16:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.476121 5095 raft_consensus.cc:729] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fae20cf6ccad4c9ba2616cc1d6e85b5f, State: Initialized, Role: FOLLOWER
08:16:10.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.476898 5095 consensus_queue.cc:260] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:10.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.480310 5105 sys_catalog.cc:455] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } } }
08:16:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.480975 5105 sys_catalog.cc:458] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [sys.catalog]: This master's current role is: FOLLOWER
08:16:10.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.483268 5095 sys_catalog.cc:564] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [sys.catalog]: configured and running, proceeding with master startup.
08:16:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551370, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:16:10.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:10.504 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc
08:16:10.513 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:16:10.514 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc
08:16:10.515 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:16:10.519 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:16:10.521 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:16:10.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:16:10.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.511024 5015 master_runner.cc:186] Error getting master registration for 127.4.184.126:45717: OK, instance_id { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" instance_seqno: 1714551366574834 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:16:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.557276 5015 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:10.543490 5117 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:10.561681 5117 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:10.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:10 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551370, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:10.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:16:10.575 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:10.575 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc
08:16:10.577 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:16:10.577 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.184.125:38831
08:16:10.579 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:16:10.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.579919 4833 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 4937
08:16:10.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:10.625 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.184.124:43649
08:16:10.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:16:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.626564 4833 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 5015
08:16:10.662 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:10.662 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.184.126:45717
08:16:10.662 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:16:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.663178 4833 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 4861
08:16:10.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:10.686 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.184.125:38831
08:16:10.688 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:16:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:10.690462 4833 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/logs
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/info.pb
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.125:38831
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.125
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34243
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.125.keytab
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.125
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:10.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:10.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:10.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:10.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:11.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.013072 5122 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:11.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.013715 5122 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:11.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.014324 5122 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.046015 5122 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.046265 5122 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.046531 5122 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.046751 5122 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.046962 5122 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.047185 5122 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.047329 5122 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44881/jwks.json
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:11.083793 5122 master_runner.cc:386] Master server non-default flags:
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.125.keytab
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.125
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.125:38831
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/info.pb
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.125
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34243
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/logs
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:11.084928 5122 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:11.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:11.085978 5122 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.101627 5130 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.104737 5132 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:11.106117 5122 server_base.cc:1034] running on GCE node
08:16:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:11.103636 5129 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:12.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.265113 5122 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:12.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.268369 5122 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:12.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.269869 5122 hybrid_clock.cc:648] HybridClock initialized: now 1714551372269823 us; error 63 us; skew 500 ppm
08:16:12.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.270758 5122 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:12.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:12 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551372, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.281983 5122 init.cc:377] Logged in from keytab as kudu/127.4.184.125@KRBTEST.COM (short username kudu)
08:16:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.287773 5122 webserver.cc:460] Webserver started at http://127.4.184.125:34243/ using document root <none> and password file <none>
08:16:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.288750 5122 fs_manager.cc:359] Metadata directory not provided
08:16:12.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.288964 5122 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.297221 5122 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
08:16:12.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.301666 5142 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.302749 5122 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:16:12.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.303182 5122 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:12.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a167916b70e74b248d2ab3228786e6ce"
08:16:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:08 on dist-test-slave-60sj"
08:16:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.305797 5122 fs_report.cc:383] FS layout report
08:16:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal
08:16:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/data
08:16:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:12.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.361768 5122 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:12.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.367516 5122 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:12.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.367993 5122 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.439152 5122 rpc_server.cc:307] RPC server started. Bound to: 127.4.184.125:38831
08:16:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.439591 5193 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.184.125:38831 every 8 connection(s)
08:16:12.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.441965 5122 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/data/info.pb
08:16:12.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.447331 4833 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 5122
08:16:12.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:12.448 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.184.124:43649
08:16:12.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.449707 4833 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/logs
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/info.pb
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.124:43649
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.124
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44329
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.124.keytab
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.124
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:12.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:12.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.456190 5194 sys_catalog.cc:263] Verifying existing consensus state
08:16:12.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240501 08:16:12.457192 5194 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:16:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.457731 5122 master.cc:537] Master@127.4.184.125:38831 shutting down...
08:16:12.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.518653 5122 master.cc:552] Master@127.4.184.125:38831 shutdown complete.
08:16:12.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:16:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.777513 5196 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.778194 5196 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.778780 5196 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.810086 5196 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.810325 5196 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.810586 5196 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.810791 5196 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.810981 5196 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.811246 5196 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.811492 5196 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44881/jwks.json
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.848642 5196 master_runner.cc:386] Master server non-default flags:
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.124.keytab
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.124
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.124:43649
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/info.pb
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.124
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44329
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/logs
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:16:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:16:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:16:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.850016 5196 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.851419 5196 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:12.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.868160 5202 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:14.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.869917 5203 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:14.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:12.872305 5196 server_base.cc:1034] running on GCE node
08:16:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:12.871847 5205 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.025703 5196 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:14.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.028388 5196 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.029744 5196 hybrid_clock.cc:648] HybridClock initialized: now 1714551374029718 us; error 47 us; skew 500 ppm
08:16:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.030623 5196 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:14.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:14 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551374, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.037110 5196 init.cc:377] Logged in from keytab as kudu/127.4.184.124@KRBTEST.COM (short username kudu)
08:16:14.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.042105 5196 webserver.cc:460] Webserver started at http://127.4.184.124:44329/ using document root <none> and password file <none>
08:16:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.043130 5196 fs_manager.cc:359] Metadata directory not provided
08:16:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.043393 5196 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:14.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.051646 5196 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
08:16:14.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.056341 5215 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.057518 5196 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
08:16:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.057897 5196 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f"
08:16:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:10 on dist-test-slave-60sj"
08:16:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.060279 5196 fs_report.cc:383] FS layout report
08:16:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/wal
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/data
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:14.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.114431 5196 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:14.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.120551 5196 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:14.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.121155 5196 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.194952 5196 rpc_server.cc:307] RPC server started. Bound to: 127.4.184.124:43649
08:16:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.195091 5266 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.184.124:43649 every 8 connection(s)
08:16:14.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.197878 5196 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-2/data/info.pb
08:16:14.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.201035 4833 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 5196
08:16:14.202 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:14.202 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.184.126:45717
08:16:14.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:16:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.204309 4833 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0YutnF/build/tsan/bin/kudu
08:16:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/logs
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/info.pb
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.126:45717
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.126
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35137
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.126.keytab
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.126
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.209872 5267 sys_catalog.cc:263] Verifying existing consensus state
08:16:14.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.215718 5267 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: Bootstrap starting.
08:16:14.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.249976 5267 log.cc:824] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: Log is configured to *not* fsync() on all Append() calls
08:16:14.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.254633 5267 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:16:14.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.255280 5267 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: Bootstrap complete.
08:16:14.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.279922 5267 raft_consensus.cc:348] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:14.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.282635 5267 raft_consensus.cc:374] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:16:14.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.282984 5267 raft_consensus.cc:729] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fae20cf6ccad4c9ba2616cc1d6e85b5f, State: Initialized, Role: FOLLOWER
08:16:14.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.283838 5267 consensus_queue.cc:260] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:14.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.286799 5271 sys_catalog.cc:455] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } } }
08:16:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.287703 5271 sys_catalog.cc:458] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [sys.catalog]: This master's current role is: FOLLOWER
08:16:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.288607 5267 sys_catalog.cc:564] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [sys.catalog]: configured and running, proceeding with master startup.
08:16:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.333882 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.334260 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:14.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.331012 5196 master_runner.cc:186] Error getting master registration for 127.4.184.126:45717: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111),
08:16:14.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.366034 5196 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:14.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.377027 5221 tablet.cc:2369] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:16:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.553373 5269 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:14.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.554405 5269 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:14.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.555264 5269 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:14.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.589437 5269 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:14.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.589537 5271 raft_consensus.cc:482] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.590148 5269 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.590121 5271 raft_consensus.cc:504] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:14.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.590906 5269 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:14.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.591359 5269 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:14.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.591776 5269 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.592202 5269 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.592561 5269 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44881/jwks.json
08:16:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.596405 5218 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111)
08:16:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.597728 5271 leader_election.cc:288] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717), a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831)
08:16:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.617785 5218 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717): Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111)
08:16:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.620810 5217 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831): Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:14.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.621595 5217 leader_election.cc:302] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fae20cf6ccad4c9ba2616cc1d6e85b5f; no voters: 2ac666bfd8b5447282e0bf43f7de44e9, a167916b70e74b248d2ab3228786e6ce
08:16:14.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.622807 5271 raft_consensus.cc:2726] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.654203 5269 master_runner.cc:386] Master server non-default flags:
08:16:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.184.84:39283
08:16:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.184.126:45717,127.4.184.125:38831,127.4.184.124:43649
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/krb5kdc/kudu_127.4.184.126.keytab
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.184.126
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.184.126:45717
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44881/jwks.json
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/info.pb
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.184.126
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35137
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/logs
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:25 UTC on 1a15035a6830
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 72
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.655829 5269 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:14.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:14.657236 5269 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.677073 5292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:15.343544 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:15.346215 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:15.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.679009 5293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:15.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:14.690433 5295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:15.997884 5294 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1319 milliseconds
08:16:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:15.998008 5269 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:16:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.000164 5269 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:16.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.003644 5269 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.005949 5269 hybrid_clock.cc:648] HybridClock initialized: now 1714551376005866 us; error 88 us; skew 500 ppm
08:16:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.007154 5269 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:16 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551376, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.015532 5269 init.cc:377] Logged in from keytab as kudu/127.4.184.126@KRBTEST.COM (short username kudu)
08:16:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.026046 5269 webserver.cc:460] Webserver started at http://127.4.184.126:35137/ using document root <none> and password file <none>
08:16:16.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.027161 5269 fs_manager.cc:359] Metadata directory not provided
08:16:16.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.027386 5269 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.039728 5269 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
08:16:16.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.046100 5304 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.048507 5269 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.004s sys 0.000s
08:16:16.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.048985 5269 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data,/tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:16.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ac666bfd8b5447282e0bf43f7de44e9"
08:16:16.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:16:06 on dist-test-slave-60sj"
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.052588 5269 fs_report.cc:383] FS layout report
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/data
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:16.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:16.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:16.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.125927 5269 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:16.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.141351 5269 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:16.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.141935 5269 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:16.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.262521 5269 rpc_server.cc:307] RPC server started. Bound to: 127.4.184.126:45717
08:16:16.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.262706 5356 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.184.126:45717 every 8 connection(s)
08:16:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.265697 5269 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/data/info.pb
08:16:16.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.276134 4833 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0YutnF/build/tsan/bin/kudu as pid 5269
08:16:16.276 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:16.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.280581 5357 sys_catalog.cc:263] Verifying existing consensus state
08:16:16.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240501 08:16:16.281723 5357 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:16:16.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.282465 5269 master.cc:537] Master@127.4.184.126:45717 shutting down...
08:16:16.346 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:16.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:16.350806 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:16.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:16.351148 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.363898 5269 master.cc:552] Master@127.4.184.126:45717 shutdown complete.
08:16:16.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0YutnF/test-tmp/mini-kudu-cluster4814458502002347768/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:16:16.428 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.184.124:43649(127.4.184.124:43649)] connecting to peer
08:16:16.434 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:16.467 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.184.124:43649(127.4.184.124:43649)] Successfully connected to peer
08:16:16.480 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.483 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.578 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:16:16.595 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.184.124
08:16:16.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.617018 5364 raft_consensus.cc:482] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:16.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.617779 5364 raft_consensus.cc:504] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:16.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.632573 5364 leader_election.cc:288] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717), a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831)
08:16:16.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:16.650523 5218 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717): Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111)
08:16:16.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:16.656971 5217 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831): Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:16.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.663036 5217 leader_election.cc:302] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fae20cf6ccad4c9ba2616cc1d6e85b5f; no voters: 2ac666bfd8b5447282e0bf43f7de44e9, a167916b70e74b248d2ab3228786e6ce
08:16:16.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:16.664105 5364 raft_consensus.cc:2726] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:16.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:16 dist-test-slave-60sj krb5kdc[4845](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551370, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.184.124@KRBTEST.COM
08:16:16.869 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x0765b4e1, L:/127.0.0.1:40646 - R:/127.4.184.124:43649] using SASL/GSSAPI
08:16:16.888 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:16.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:16.907 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:16.911 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.911 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.914 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.917 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.917 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.919 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:16.922 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:16.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:16.925 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.928 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.930 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.930 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:16.942 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:16.944 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:16.946 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.946 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.947 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.947 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.947 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.949 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:16.982 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:16.984 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:16.986 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.986 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.987 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:16.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:16.991 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:17.022 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:17.024 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:17.026 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.027 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.027 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.028 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.031 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:17.062 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:17.064 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:17.067 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.067 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.068 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.071 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:17.082 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:17.084 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:17.086 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.086 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.091 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.091 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.093 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:17.282 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:17.285 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:17.287 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.287 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.290 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.291 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.291 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.291 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:17.354071 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:17.354346 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:17.442 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:17.444 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:17.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.446 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:17.447 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:17.449 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:18.022 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:18.024 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:18.025 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:18.025 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:18.025 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:18.025 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:18.026 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:18.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:18.027 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:18.357108 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:18.357370 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:18.842 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:18.844 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:18.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:18.846 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:18.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:18.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:18.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:18.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:18.848 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:19.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:19.089591 5371 raft_consensus.cc:482] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:19.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:19.089980 5371 raft_consensus.cc:504] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:19.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:19.091532 5371 leader_election.cc:288] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717), a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831)
08:16:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:19.097410 5218 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717): Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111)
08:16:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:19.098084 5217 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831): Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:19.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:19.098385 5217 leader_election.cc:302] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fae20cf6ccad4c9ba2616cc1d6e85b5f; no voters: 2ac666bfd8b5447282e0bf43f7de44e9, a167916b70e74b248d2ab3228786e6ce
08:16:19.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:19.098904 5371 raft_consensus.cc:2726] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:19.202 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:19.203 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:19.206 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:19.207 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:19.207 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:19.207 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:19.207 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:19.208 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:19.208 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:19.360208 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:19.360518 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:19.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:19.368373 5196 master_runner.cc:186] Error getting master registration for 127.4.184.126:45717: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111),
08:16:19.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:19.368722 5196 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:20.363471 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:20.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:20.363739 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:20.942 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:20.943 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:20.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:20.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:20.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:20.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:20.946 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:20.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:20.947 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:21.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:21.090116 5379 raft_consensus.cc:482] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:21.090610 5379 raft_consensus.cc:504] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:21.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:21.092721 5379 leader_election.cc:288] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717), a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831)
08:16:21.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:21.094868 5218 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111) [suppressed 5 similar messages]
08:16:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:21.097858 5217 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831): Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:21.098412 5218 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717): Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111)
08:16:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:21.098724 5218 leader_election.cc:302] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fae20cf6ccad4c9ba2616cc1d6e85b5f; no voters: 2ac666bfd8b5447282e0bf43f7de44e9, a167916b70e74b248d2ab3228786e6ce
08:16:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:21.099292 5379 raft_consensus.cc:2726] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:21.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:21.366537 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:21.366802 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:22.369606 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:22.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:22.369870 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:23.373620 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:23.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:23.373957 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:23.962 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:23.964 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:23.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:23.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:23.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:23.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:23.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:23.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:23.968 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:24.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:24 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551384, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:24.041049 5211 init.cc:295] Successfully reacquired a new kerberos TGT
08:16:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:24.372193 5196 master_runner.cc:186] Error getting master registration for 127.4.184.126:45717: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111),
08:16:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:24.372634 5196 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:24.376816 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:24.377068 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:24.744561 5389 raft_consensus.cc:482] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:24.745123 5389 raft_consensus.cc:504] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:24.746961 5389 leader_election.cc:288] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717), a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831)
08:16:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:24.751221 5218 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717): Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111)
08:16:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:24.751801 5217 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831): Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:24.752089 5217 leader_election.cc:302] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fae20cf6ccad4c9ba2616cc1d6e85b5f; no voters: 2ac666bfd8b5447282e0bf43f7de44e9, a167916b70e74b248d2ab3228786e6ce
08:16:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:24.752663 5389 raft_consensus.cc:2726] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:25.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:25.379848 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:25.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:25.380110 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:26.049000 4678 debug-util.cc:398] Leaking SignalData structure 0x7b08000be220 after lost signal to thread 4552
08:16:26.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:26.050055 4678 debug-util.cc:398] Leaking SignalData structure 0x7b08000be260 after lost signal to thread 4681
08:16:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:26.383296 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:26.383597 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:26.762 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:26.764 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:26.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:26.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:26.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:26.767 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:26.767 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:26.767 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:26.769 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:27.386917 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:27.387188 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:28.122 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:28.124 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:28.125 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:28.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:28.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:28.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:28.126 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:28.127 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:28.128 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:28.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:28.390100 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:28.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:28.390381 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:28.502 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:28.503 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:28.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:28.505 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:28.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:28.505 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:28.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:28.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:28.507 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:29.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:29.375548 5196 master_runner.cc:186] Error getting master registration for 127.4.184.126:45717: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111),
08:16:29.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:29.375892 5196 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:29.381 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:29.383 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:29.384 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:29.384 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:29.385 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:29.386 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:29.387 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:29.387 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:29.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:29.393167 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:29.393457 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:29.502 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:29.504 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:29.506 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:29.506 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:29.506 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:29.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:29.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:29.506 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:29.507 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:30.396435 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:30.396698 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:31.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:31.399641 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:31.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:31.399909 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:32.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:32.402753 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:32.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:32.403036 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:32.641 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:32.642 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:32.643 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:32.643 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:32.644 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:32.644 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:32.644 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:32.644 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:32.646 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:33.406884 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:33.407213 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 08:16:34 dist-test-slave-60sj krb5kdc[4845](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714551394, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.184.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:34.043658 5211 init.cc:295] Successfully reacquired a new kerberos TGT
08:16:34.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:34.379285 5196 master_runner.cc:186] Error getting master registration for 127.4.184.126:45717: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111),
08:16:34.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:34.379716 5196 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:34.410177 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:34.410463 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:34.535786 5405 raft_consensus.cc:482] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:34.536346 5405 raft_consensus.cc:504] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:34.538148 5405 leader_election.cc:288] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717), a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831)
08:16:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:34.539629 5218 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:16:34.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:34.542709 5218 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717): Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111)
08:16:34.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:34.543288 5217 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831): Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:34.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:34.543627 5217 leader_election.cc:302] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fae20cf6ccad4c9ba2616cc1d6e85b5f; no voters: 2ac666bfd8b5447282e0bf43f7de44e9, a167916b70e74b248d2ab3228786e6ce
08:16:34.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:34.544291 5405 raft_consensus.cc:2726] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:35.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:35.413795 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:35.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:35.414078 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:36.417407 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:36.417702 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:36.602 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:36.603 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:36.604 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:36.604 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:36.604 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:36.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:36.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:36.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:36.608 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:37.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:37.420962 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:37.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:37.421221 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:37.762 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:37.763 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:37.766 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:37.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:37.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:37.767 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:37.768 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:37.768 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:37.772 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:38.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:38.424086 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:38.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:38.424347 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:38.882 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:38.883 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:38.884 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:38.884 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:38.884 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:38.884 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:38.884 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:38.884 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:38.886 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:39.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:39.382725 5196 master_runner.cc:186] Error getting master registration for 127.4.184.126:45717: Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111),
08:16:39.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:39.383196 5196 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:39.427707 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:39.427982 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:39.518384 5411 raft_consensus.cc:482] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:39.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:39.518760 5411 raft_consensus.cc:504] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ac666bfd8b5447282e0bf43f7de44e9" member_type: VOTER last_known_addr { host: "127.4.184.126" port: 45717 } } peers { permanent_uuid: "a167916b70e74b248d2ab3228786e6ce" member_type: VOTER last_known_addr { host: "127.4.184.125" port: 38831 } } peers { permanent_uuid: "fae20cf6ccad4c9ba2616cc1d6e85b5f" member_type: VOTER last_known_addr { host: "127.4.184.124" port: 43649 } }
08:16:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:39.520272 5411 leader_election.cc:288] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717), a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831)
08:16:39.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:39.524017 5218 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ac666bfd8b5447282e0bf43f7de44e9 (127.4.184.126:45717): Network error: Client connection negotiation failed: client connection to 127.4.184.126:45717: connect: Connection refused (error 111)
08:16:39.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:39.524400 5217 leader_election.cc:334] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a167916b70e74b248d2ab3228786e6ce (127.4.184.125:38831): Network error: Client connection negotiation failed: client connection to 127.4.184.125:38831: connect: Connection refused (error 111)
08:16:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:39.524698 5217 leader_election.cc:302] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fae20cf6ccad4c9ba2616cc1d6e85b5f; no voters: 2ac666bfd8b5447282e0bf43f7de44e9, a167916b70e74b248d2ab3228786e6ce
08:16:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:39.525226 5411 raft_consensus.cc:2726] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:40.431507 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:40.431759 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:40.462 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:40.463 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:40.464 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:40.464 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:40.464 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:40.464 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:40.464 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:40.464 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:40.468 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:41.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:41.434814 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:41.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:41.435134 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:42.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:42.438001 5282 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fae20cf6ccad4c9ba2616cc1d6e85b5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:42.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:16:42.438395 5282 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:43.102 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.125:38831(127.4.184.125:38831)] connecting to peer
08:16:43.102 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.184.126:45717(127.4.184.126:45717)] connecting to peer
08:16:43.103 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:43.103 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.184.126:45717(127.4.184.126:45717)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:43.103 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:43.104 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.125:38831(127.4.184.125:38831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:43.104 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.126:45717: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717
08:16:43.104 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.184.125:38831: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
08:16:43.107 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717; will retry
08:16:43.112 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26827), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [197ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [201ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [631ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [633ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [635ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [646ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [661ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [662ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [663ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [701ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [704ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [706ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [782ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [784ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [805ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [807ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [809ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1007ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1157ms] refreshing cache from master, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1737ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1742ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2557ms] refreshing cache from master, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [2563ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2922ms] too many traces: truncated at 100 traces, deferred=Deferred@977238006(state=PENDING, result=null, callback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980), errback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980)))
08:16:43.117 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.184.126:45717(127.4.184.126:45717)] disconnecting while in state TERMINATED
08:16:43.118 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.184.124:43649(127.4.184.124:43649)] disconnecting while in state READY
08:16:43.118 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.184.125:38831(127.4.184.125:38831)] disconnecting while in state TERMINATED
08:16:43.124 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:16:43.126 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.184.124:43649(127.4.184.124:43649)] handling channelInactive
08:16:43.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.184.124:43649(127.4.184.124:43649)] cleaning up while in state READY due to: connection closed
08:16:43.127 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:16:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:43.130905 4833 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 5196
08:16:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:16:44Z chronyd exiting
08:16:45.275 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:16:45.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:45.277007 4109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 4388
08:16:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:45.303104 4109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 4545
08:16:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:45.331868 4109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 4691
08:16:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:45.357235 4109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 4121
08:16:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:45.389997 4109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 4191
08:16:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:16:45.423214 4109 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0YutnF/build/tsan/bin/kudu with pid 4266
08:16:46.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:16:46Z chronyd exiting
08:16:47.529 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26827), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [197ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [201ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [631ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [633ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [635ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [646ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [661ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [662ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [663ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [701ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [704ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [706ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [782ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [784ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [805ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [807ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [809ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1007ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1157ms] refreshing cache from master, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1737ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1742ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2557ms] refreshing cache from master, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [2563ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2922ms] too many traces: truncated at 100 traces, deferred=Deferred@977238006(state=PENDING, result=null, callback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980), errback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.184.125:38831
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26827), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [197ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [201ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [631ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [633ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [635ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [646ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [661ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [662ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [663ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [701ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [704ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [706ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [782ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [784ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [805ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [807ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [809ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1007ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1157ms] refreshing cache from master, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1737ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1742ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2557ms] refreshing cache from master, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [2563ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2922ms] too many traces: truncated at 100 traces, deferred=Deferred@977238006(state=PENDING, result=null, callback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980), errback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.184.125:38831
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
08:16:47.993 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
08:16:47.993 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26827), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [197ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [201ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [631ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [633ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [635ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [646ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [661ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [662ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [663ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [701ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [704ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [706ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [782ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [784ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [805ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [807ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [809ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1007ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1157ms] refreshing cache from master, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1737ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1742ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2557ms] refreshing cache from master, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [2563ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2922ms] too many traces: truncated at 100 traces, deferred=Deferred@977238006(state=PENDING, result=null, callback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980), errback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.184.125:38831
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26827), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [197ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [201ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [599ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [617ms] refreshing cache from master, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [626ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [631ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [633ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [635ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [646ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [661ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [662ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [663ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [701ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [704ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [706ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [707ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [782ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [784ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [802ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [805ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [807ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [809ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1006ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1007ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1157ms] refreshing cache from master, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1163ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1164ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1737ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [1741ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [1742ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [1742ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2557ms] refreshing cache from master, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.126:45717: Network error: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.125:38831: Network error: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.4.184.124:43649: OK, [2563ms] delaying RPC due to: Service unavailable: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.125:38831, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.124:43649, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.184.126:45717, [2922ms] too many traces: truncated at 100 traces, deferred=Deferred@977238006(state=PENDING, result=null, callback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980), errback=(continuation of Deferred@1604972470 after retry RPC after error@344306332) -> (continuation of Deferred@987941645 after retry RPC after error@2074315616) -> (continuation of Deferred@1810309117 after retry RPC after error@1389318479) -> (continuation of Deferred@2119779000 after retry RPC after error@424033944) -> (continuation of Deferred@1242415802 after retry RPC after error@1650505629) -> (continuation of Deferred@1452828139 after retry RPC after error@2122457950) -> (continuation of Deferred@1514026534 after retry RPC after error@1023786659) -> (continuation of Deferred@673653717 after retry RPC after error@1196453212) -> (continuation of Deferred@1819606166 after retry RPC after error@2081728285) -> (continuation of Deferred@1832651457 after retry RPC after error@692232375) -> (continuation of Deferred@1645832517 after retry RPC after error@996935252) -> (continuation of Deferred@906717690 after retry RPC after error@183513236) -> (continuation of Deferred@587871182 after retry RPC after error@2014302045) -> (continuation of Deferred@354827551 after retry RPC after error@509205439) -> (continuation of Deferred@1824460793 after retry RPC after error@1791774740) -> (continuation of Deferred@1311135383 after retry RPC after error@510851279) -> (continuation of Deferred@170323007 after retry RPC after error@39997855) -> (continuation of Deferred@2123115946 after retry RPC after error@1999705652) -> (continuation of Deferred@1246448633 after retry RPC after error@1521812870) -> (continuation of Deferred@1162580487 after retry RPC after error@976530476) -> (continuation of Deferred@707800080 after retry RPC after error@1482397157) -> (continuation of Deferred@1100607299 after retry RPC after error@1590918800) -> (continuation of Deferred@1973979638 after retry RPC after error@1910033947) -> (continuation of Deferred@2142618772 after retry RPC after error@978784780) -> (continuation of Deferred@193059721 after retry RPC after error@1008628980)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.184.125:38831,127.4.184.124:43649,127.4.184.126:45717) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.126:45717(127.4.184.126:45717): Connection refused: /127.4.184.126:45717,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.184.125:38831(127.4.184.125:38831): Connection refused: /127.4.184.125:38831
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.184.125:38831
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)