07:06:21.582 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:06:21.596 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLw1kUt/test-tmp
07:06:21.598 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:06:21.603 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLw1kUt/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:06:21.861 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:06:22.017 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:06:22.019 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:06:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.024996 11982 mini_oidc.cc:152] Starting JWKS server
07:06:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.031127 11982 webserver.cc:460] Webserver started at http://127.0.0.1:39053/ using document root <none> and password file <none>
07:06:22.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.031756 11982 mini_oidc.cc:189] Starting OIDC Discovery server
07:06:22.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.034050 11982 webserver.cc:460] Webserver started at http://127.0.0.1:35595/ using document root <none> and password file <none>
07:06:22.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:06:22Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:06:22.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:06:22Z Disabled control of system clock
07:06:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.107576 11982 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/wal
07:06:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/data
07:06:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/logs
07:06:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/data/info.pb
07:06:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.190:33769
07:06:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.190
07:06:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39053/jwks.json with env {}
07:06:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.438436 11994 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.439054 11994 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.439682 11994 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:22.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.471287 11994 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.471597 11994 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.471810 11994 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.472074 11994 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.472390 11994 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.472640 11994 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39053/jwks.json
07:06:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.517282 11994 master_runner.cc:386] Master server non-default flags:
07:06:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/data
07:06:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/wal
07:06:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.190:33769
07:06:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39053/jwks.json
07:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/data/info.pb
07:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.190
07:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/logs
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.518738 11994 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.521227 11994 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.541033 12003 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:23.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.541564 12000 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:22.542273 12001 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:22.542816 11994 server_base.cc:1034] running on GCE node
07:06:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.663002 11994 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.666222 11994 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:23.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.667687 11994 hybrid_clock.cc:648] HybridClock initialized: now 1714115183667631 us; error 77 us; skew 500 ppm
07:06:23.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.668677 11994 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.676600 11994 webserver.cc:460] Webserver started at http://127.11.179.190:46339/ using document root <none> and password file <none>
07:06:23.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.677870 11994 fs_manager.cc:359] Metadata directory not provided
07:06:23.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.678102 11994 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:23.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.678535 11994 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:23.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.683965 11994 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/data/instance:
07:06:23.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a238c6855330413db6f1304e16020bc0"
07:06:23.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:23 on dist-test-slave-9jsd"
07:06:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.685623 11994 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/wal/instance:
07:06:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a238c6855330413db6f1304e16020bc0"
07:06:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:23 on dist-test-slave-9jsd"
07:06:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.694264 11994 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
07:06:23.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.701464 12010 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.702879 11994 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
07:06:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.703274 11994 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/wal
07:06:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a238c6855330413db6f1304e16020bc0"
07:06:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:23 on dist-test-slave-9jsd"
07:06:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.703649 11994 fs_report.cc:383] FS layout report
07:06:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/wal
07:06:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/wal
07:06:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/data/data
07:06:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:23.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.764289 11994 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:23.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.765965 11994 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:23.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.766456 11994 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:23.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.849927 11994 rpc_server.cc:307] RPC server started. Bound to: 127.11.179.190:33769
07:06:23.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.850030 12061 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.179.190:33769 every 8 connection(s)
07:06:23.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.853365 11994 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/data/info.pb
07:06:23.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.859637 12062 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:06:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.862516 11982 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 11994
07:06:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.862959 11982 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-0/wal/instance
07:06:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.866851 11982 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/wal
07:06:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/data
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/logs
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/data/info.pb
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:23.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.189:34495
07:06:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.189
07:06:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39053/jwks.json with env {}
07:06:23.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.875967 12062 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } has no permanent_uuid. Determining permanent_uuid...
07:06:23.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.909206 12062 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } has no permanent_uuid. Determining permanent_uuid...
07:06:23.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:23.912281 12014 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.179.189:34495: connect: Connection refused (error 111)
07:06:23.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:23.917141 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.189:34495: Network error: Client connection negotiation failed: client connection to 127.11.179.189:34495: connect: Connection refused (error 111)
07:06:23.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:23.966758 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } attempt: 1
07:06:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:23.972312 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.189:34495: Network error: Client connection negotiation failed: client connection to 127.11.179.189:34495: connect: Connection refused (error 111)
07:06:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:24.040766 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } attempt: 2
07:06:24.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.045737 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.189:34495: Network error: Client connection negotiation failed: client connection to 127.11.179.189:34495: connect: Connection refused (error 111)
07:06:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:24.137217 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } attempt: 3
07:06:24.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.141577 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.189:34495: Network error: Client connection negotiation failed: client connection to 127.11.179.189:34495: connect: Connection refused (error 111)
07:06:24.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.208784 12064 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:24.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.209477 12064 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.210147 12064 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.243168 12064 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.243485 12064 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:24.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.243686 12064 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.243875 12064 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.244058 12064 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.244210 12064 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39053/jwks.json
07:06:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:24.279875 12064 master_runner.cc:386] Master server non-default flags:
07:06:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/data
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/wal
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.189:34495
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39053/jwks.json
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/data/info.pb
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.189
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/logs
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:24.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:24.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:24.281455 12064 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:24.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:24.283980 12064 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:24.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:24.285123 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } attempt: 4
07:06:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.289933 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.189:34495: Network error: Client connection negotiation failed: client connection to 127.11.179.189:34495: connect: Connection refused (error 111)
07:06:24.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.305558 12076 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:24.589457 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } attempt: 5
07:06:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.595304 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.189:34495: Network error: Client connection negotiation failed: client connection to 127.11.179.189:34495: connect: Connection refused (error 111)
07:06:25.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.142841 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } attempt: 6
07:06:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.150585 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.189:34495: Network error: Client connection negotiation failed: client connection to 127.11.179.189:34495: connect: Connection refused (error 111)
07:06:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.305821 12073 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:24.306138 12074 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:24.307114 12064 server_base.cc:1034] running on GCE node
07:06:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.438714 12064 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.442428 12064 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.443967 12064 hybrid_clock.cc:648] HybridClock initialized: now 1714115185443910 us; error 69 us; skew 500 ppm
07:06:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.445106 12064 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:25.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.452529 12064 webserver.cc:460] Webserver started at http://127.11.179.189:43501/ using document root <none> and password file <none>
07:06:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.453729 12064 fs_manager.cc:359] Metadata directory not provided
07:06:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.453969 12064 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.454391 12064 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.459961 12064 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/data/instance:
07:06:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f501d1eaf4147289eee2c0861c9a1fd"
07:06:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:25 on dist-test-slave-9jsd"
07:06:25.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.461364 12064 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/wal/instance:
07:06:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f501d1eaf4147289eee2c0861c9a1fd"
07:06:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:25 on dist-test-slave-9jsd"
07:06:25.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.469923 12064 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
07:06:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.476895 12085 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:25.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.478744 12064 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
07:06:25.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.479216 12064 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/wal
07:06:25.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f501d1eaf4147289eee2c0861c9a1fd"
07:06:25.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:25 on dist-test-slave-9jsd"
07:06:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.479705 12064 fs_report.cc:383] FS layout report
07:06:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/wal
07:06:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/wal
07:06:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/data/data
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:25.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.536000 12064 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.537825 12064 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.538414 12064 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.619072 12064 rpc_server.cc:307] RPC server started. Bound to: 127.11.179.189:34495
07:06:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.619194 12136 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.179.189:34495 every 8 connection(s)
07:06:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.622288 12064 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/data/info.pb
07:06:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.628290 12137 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:06:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.631131 11982 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 12064
07:06:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.631641 11982 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-1/wal/instance
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.634783 11982 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/wal
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/data
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/logs
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/data/info.pb
07:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.188:39405
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.188
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:25.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39053/jwks.json with env {}
07:06:25.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.645013 12137 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } has no permanent_uuid. Determining permanent_uuid...
07:06:25.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.669744 12137 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } has no permanent_uuid. Determining permanent_uuid...
07:06:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.687127 12137 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } has no permanent_uuid. Determining permanent_uuid...
07:06:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.689754 12088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:25.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.695338 12137 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:25.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.745026 12137 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 1
07:06:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.750008 12137 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:25.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.818459 12137 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 2
07:06:25.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.823146 12137 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:25.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:25.914613 12137 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 3
07:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.919052 12137 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:25.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.977581 12139 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:25.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.978186 12139 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:25.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:25.978734 12139 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.009711 12139 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.010023 12139 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.010228 12139 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.010416 12139 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.010615 12139 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.010762 12139 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39053/jwks.json
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.046662 12139 master_runner.cc:386] Master server non-default flags:
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/data
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/wal
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.188:39405
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39053/jwks.json
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/data/info.pb
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.188
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/logs
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.048167 12139 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:26.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.050598 12139 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:26.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.062489 12137 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 4
07:06:26.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.069574 12137 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.070984 12152 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.211329 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } attempt: 7
07:06:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.225067 12062 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } has no permanent_uuid. Determining permanent_uuid...
07:06:26.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.233985 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:26.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.292486 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 1
07:06:26.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.299396 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.369077 12137 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 5
07:06:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.373468 12137 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:26.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.380894 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 2
07:06:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.388069 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:26.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.473560 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 3
07:06:26.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.480409 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.620893 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 4
07:06:26.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.628159 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:26.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.911811 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 5
07:06:26.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.918300 12062 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.921336 12137 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 6
07:06:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.929647 12137 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.179.188:39405: Network error: Client connection negotiation failed: client connection to 127.11.179.188:39405: connect: Connection refused (error 111)
07:06:27.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.071333 12149 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:27.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:26.071993 12150 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:27.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:26.072409 12139 server_base.cc:1034] running on GCE node
07:06:27.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.216573 12139 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:27.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.219657 12139 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:27.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.221112 12139 hybrid_clock.cc:648] HybridClock initialized: now 1714115187221079 us; error 53 us; skew 500 ppm
07:06:27.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.222084 12139 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:27.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.228956 12139 webserver.cc:460] Webserver started at http://127.11.179.188:42889/ using document root <none> and password file <none>
07:06:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.230113 12139 fs_manager.cc:359] Metadata directory not provided
07:06:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.230342 12139 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.230774 12139 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.236788 12139 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/data/instance:
07:06:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f30b13bd571447a96429157a43be51b"
07:06:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:27 on dist-test-slave-9jsd"
07:06:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.238149 12139 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/wal/instance:
07:06:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f30b13bd571447a96429157a43be51b"
07:06:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:27 on dist-test-slave-9jsd"
07:06:27.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.246970 12139 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.000s sys 0.008s
07:06:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.253641 12164 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.255009 12139 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:06:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.255439 12139 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/wal
07:06:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f30b13bd571447a96429157a43be51b"
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:27 on dist-test-slave-9jsd"
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.255821 12139 fs_report.cc:383] FS layout report
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/wal
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/wal
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/data/data
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:27.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:27.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:27.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:27.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.311407 12139 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.313225 12139 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.313766 12139 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.394542 12139 rpc_server.cc:307] RPC server started. Bound to: 127.11.179.188:39405
07:06:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.394680 12215 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.179.188:39405 every 8 connection(s)
07:06:27.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.397639 12139 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/data/info.pb
07:06:27.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.399170 11982 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 12139
07:06:27.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.399767 11982 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/master-2/wal/instance
07:06:27.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.404760 12216 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:06:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.431089 12216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } has no permanent_uuid. Determining permanent_uuid...
07:06:27.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.465051 12216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } has no permanent_uuid. Determining permanent_uuid...
07:06:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.470798 12062 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 6
07:06:27.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.477770 12216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } has no permanent_uuid. Determining permanent_uuid...
07:06:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.496286 12062 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: Bootstrap starting.
07:06:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.502559 12216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b: Bootstrap starting.
07:06:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.503661 12062 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: Neither blocks nor log segments found. Creating new log.
07:06:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.505862 12062 log.cc:824] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: Log is configured to *not* fsync() on all Append() calls
07:06:27.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.511525 12216 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b: Neither blocks nor log segments found. Creating new log.
07:06:27.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.514580 12216 log.cc:824] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b: Log is configured to *not* fsync() on all Append() calls
07:06:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.516772 12062 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: No bootstrap required, opened a new log
07:06:27.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.520917 12216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b: No bootstrap required, opened a new log
07:06:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.553228 12062 raft_consensus.cc:348] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.554198 12062 raft_consensus.cc:374] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:06:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.554498 12062 raft_consensus.cc:729] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a238c6855330413db6f1304e16020bc0, State: Initialized, Role: FOLLOWER
07:06:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.555474 12062 consensus_queue.cc:260] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.555850 12216 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.556743 12216 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:06:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.557102 12216 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f30b13bd571447a96429157a43be51b, State: Initialized, Role: FOLLOWER
07:06:27.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.558863 12228 sys_catalog.cc:455] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:27.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.558714 12216 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:27.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.559688 12228 sys_catalog.cc:458] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [sys.catalog]: This master's current role is: FOLLOWER
07:06:27.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.561472 12062 sys_catalog.cc:564] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [sys.catalog]: configured and running, proceeding with master startup.
07:06:27.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.562192 12229 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:27.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.563532 12229 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: This master's current role is: FOLLOWER
07:06:27.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.564369 12216 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: configured and running, proceeding with master startup.
07:06:27.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:27.569703 12170 tablet.cc:2369] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:06:27.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:27.605633 12251 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:27.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:27.606496 12251 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:27.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:27.617482 12250 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:27.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:27.629297 12250 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.599777 11994 master_runner.cc:186] Error getting master registration for 127.11.179.189:34495: OK, instance_id { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" instance_seqno: 1714115185596577 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:06:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.644359 11994 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:06:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.610679 12139 master_runner.cc:186] Error getting master registration for 127.11.179.189:34495: OK, instance_id { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" instance_seqno: 1714115185596577 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:06:27.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.657918 12139 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:06:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.681252 12229 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:06:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.681989 12229 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.685109 12229 leader_election.cc:288] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a238c6855330413db6f1304e16020bc0 (127.11.179.190:33769), 6f501d1eaf4147289eee2c0861c9a1fd (127.11.179.189:34495)
07:06:27.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.686434 12037 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6f30b13bd571447a96429157a43be51b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a238c6855330413db6f1304e16020bc0" is_pre_election: true
07:06:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.687556 12037 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6f30b13bd571447a96429157a43be51b in term 0.
07:06:27.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.689462 12167 leader_election.cc:302] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6f30b13bd571447a96429157a43be51b, a238c6855330413db6f1304e16020bc0; no voters:
07:06:27.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.690644 12229 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 0 FOLLOWER]: Leader pre-election won for term 1
07:06:27.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.691066 12229 raft_consensus.cc:482] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:06:27.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.691448 12229 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 0 FOLLOWER]: Advancing to term 1
07:06:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.706341 12229 raft_consensus.cc:504] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:27.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.712667 12037 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6f30b13bd571447a96429157a43be51b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a238c6855330413db6f1304e16020bc0"
07:06:27.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.713238 12229 leader_election.cc:288] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [CANDIDATE]: Term 1 election: Requested vote from peers a238c6855330413db6f1304e16020bc0 (127.11.179.190:33769), 6f501d1eaf4147289eee2c0861c9a1fd (127.11.179.189:34495)
07:06:27.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.714679 12037 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [term 0 FOLLOWER]: Advancing to term 1
07:06:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.723754 12037 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6f30b13bd571447a96429157a43be51b in term 1.
07:06:27.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.731017 12167 leader_election.cc:302] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6f30b13bd571447a96429157a43be51b, a238c6855330413db6f1304e16020bc0; no voters:
07:06:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.732368 12229 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 1 FOLLOWER]: Leader election won for term 1
07:06:27.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.747179 12229 raft_consensus.cc:686] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [term 1 LEADER]: Becoming Leader. State: Replica: 6f30b13bd571447a96429157a43be51b, State: Running, Role: LEADER
07:06:27.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.748720 12229 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.773532 12254 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6f30b13bd571447a96429157a43be51b. Latest consensus state: current_term: 1 leader_uuid: "6f30b13bd571447a96429157a43be51b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:27.776892 12016 tablet.cc:2369] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:06:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.776540 12254 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: This master's current role is: LEADER
07:06:27.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.782229 12256 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:06:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.794857 12256 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:06:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.818928 12037 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [term 1 FOLLOWER]: Refusing update from remote peer 6f30b13bd571447a96429157a43be51b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:06:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.824610 12254 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [LEADER]: Connected to new peer: Peer: permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
07:06:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.865295 12228 sys_catalog.cc:455] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6f30b13bd571447a96429157a43be51b. Latest consensus state: current_term: 1 leader_uuid: "6f30b13bd571447a96429157a43be51b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.866562 12228 sys_catalog.cc:458] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [sys.catalog]: This master's current role is: FOLLOWER
07:06:27.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.875496 12257 mvcc.cc:205] Tried to move back new op lower bound from 7021015809279451136 to 7021015809046093824. Current Snapshot: MvccSnapshot[applied={T|T < 7021015809279451136}]
07:06:27.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.882264 12228 sys_catalog.cc:455] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6f30b13bd571447a96429157a43be51b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:27.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.883107 12228 sys_catalog.cc:458] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0 [sys.catalog]: This master's current role is: FOLLOWER
07:06:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.885766 12258 mvcc.cc:205] Tried to move back new op lower bound from 7021015809279451136 to 7021015809046093824. Current Snapshot: MvccSnapshot[applied={T|T < 7021015809279451136 or (T in {7021015809279451136})}]
07:06:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.897063 12256 catalog_manager.cc:1347] Generated new cluster ID: 13d398211f594f4897a85fca04b3feb0
07:06:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.897490 12256 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:06:27.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.904807 12229 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6f30b13bd571447a96429157a43be51b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:27.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.905783 12229 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: This master's current role is: LEADER
07:06:27.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.941524 12256 catalog_manager.cc:1370] Generated new certificate authority record
07:06:27.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.943812 12256 catalog_manager.cc:1504] Loading token signing keys...
07:06:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.982297 12256 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b: Generated new TSK 0
07:06:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.988047 12256 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:06:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:27.990376 12137 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } attempt: 7
07:06:28.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.026281 11982 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:28.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/wal
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/data
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/logs
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/data/info.pb
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.129:0
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.179.129
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.129
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:28.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.032145 12137 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd: Bootstrap starting.
07:06:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.052040 12137 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd: Neither blocks nor log segments found. Creating new log.
07:06:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.059237 12137 log.cc:824] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd: Log is configured to *not* fsync() on all Append() calls
07:06:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.075346 12137 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd: No bootstrap required, opened a new log
07:06:28.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.124430 12137 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:28.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.125869 12137 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:06:28.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.126492 12137 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f501d1eaf4147289eee2c0861c9a1fd, State: Initialized, Role: FOLLOWER
07:06:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.127854 12137 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } }
07:06:28.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.142876 12265 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.146013 12265 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [sys.catalog]: This master's current role is: FOLLOWER
07:06:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.148602 12110 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [term 0 FOLLOWER]: Advancing to term 1
07:06:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.153522 12112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6f30b13bd571447a96429157a43be51b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" is_pre_election: true
07:06:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.155442 12111 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6f30b13bd571447a96429157a43be51b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f501d1eaf4147289eee2c0861c9a1fd"
07:06:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.161528 12137 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [sys.catalog]: configured and running, proceeding with master startup.
07:06:28.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.163746 12110 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [term 1 FOLLOWER]: Refusing update from remote peer 6f30b13bd571447a96429157a43be51b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:06:28.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.167804 12254 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
07:06:28.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.279804 12265 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6f30b13bd571447a96429157a43be51b. Latest consensus state: current_term: 1 leader_uuid: "6f30b13bd571447a96429157a43be51b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:28.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.281973 12273 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6f30b13bd571447a96429157a43be51b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:28.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.283025 12273 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [sys.catalog]: This master's current role is: FOLLOWER
07:06:28.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.284152 12265 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd [sys.catalog]: This master's current role is: FOLLOWER
07:06:28.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.340049 12281 catalog_manager.cc:1259] Loaded cluster ID: 13d398211f594f4897a85fca04b3feb0
07:06:28.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.340813 12281 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd: loading cluster ID for follower catalog manager: success
07:06:28.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.351624 12281 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd: acquiring CA information for follower catalog manager: success
07:06:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.360213 12281 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6f501d1eaf4147289eee2c0861c9a1fd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:06:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.627576 12286 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6f30b13bd571447a96429157a43be51b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a238c6855330413db6f1304e16020bc0" member_type: VOTER last_known_addr { host: "127.11.179.190" port: 33769 } } peers { permanent_uuid: "6f501d1eaf4147289eee2c0861c9a1fd" member_type: VOTER last_known_addr { host: "127.11.179.189" port: 34495 } } peers { permanent_uuid: "6f30b13bd571447a96429157a43be51b" member_type: VOTER last_known_addr { host: "127.11.179.188" port: 39405 } } }
07:06:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.628473 12286 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f30b13bd571447a96429157a43be51b [sys.catalog]: This master's current role is: LEADER
07:06:28.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.636500 12250 catalog_manager.cc:1259] Loaded cluster ID: 13d398211f594f4897a85fca04b3feb0
07:06:28.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.636853 12250 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: loading cluster ID for follower catalog manager: success
07:06:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.659969 12250 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: acquiring CA information for follower catalog manager: success
07:06:28.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:28.671193 12250 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a238c6855330413db6f1304e16020bc0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:06:28.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:28.779640 12263 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:28.781416 12263 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:28.782590 12263 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:28.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:28.869843 12263 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:28.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:28.876725 12263 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.179.129
07:06:29.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:29.007311 12263 tablet_server_runner.cc:78] Tablet server non-default flags:
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/data
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/wal
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.129:0
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/data/info.pb
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.129
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.179.129
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/logs
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:29.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:29.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:29.011328 12263 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:29.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:29.014706 12263 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:29.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:29.067401 12291 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:29.076851 12263 server_base.cc:1034] running on GCE node
07:06:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:29.087247 12293 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:29.077530 12290 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:30.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.686391 12263 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:30.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.703780 12263 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.705761 12263 hybrid_clock.cc:648] HybridClock initialized: now 1714115190705716 us; error 56 us; skew 500 ppm
07:06:30.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.707304 12263 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.733032 12263 webserver.cc:460] Webserver started at http://127.11.179.129:41105/ using document root <none> and password file <none>
07:06:30.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.734980 12263 fs_manager.cc:359] Metadata directory not provided
07:06:30.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.735742 12263 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:30.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.736655 12263 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.745401 12263 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/data/instance:
07:06:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "858d42d1b4ca4d94935bbb0a87483127"
07:06:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:30 on dist-test-slave-9jsd"
07:06:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.747773 12263 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/wal/instance:
07:06:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "858d42d1b4ca4d94935bbb0a87483127"
07:06:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:30 on dist-test-slave-9jsd"
07:06:30.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.769556 12263 fs_manager.cc:691] Time spent creating directory manager: real 0.017s user 0.011s sys 0.004s
07:06:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.786732 12300 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:30.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.788858 12263 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.006s sys 0.002s
07:06:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.789758 12263 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/wal
07:06:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "858d42d1b4ca4d94935bbb0a87483127"
07:06:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:30 on dist-test-slave-9jsd"
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.790743 12263 fs_report.cc:383] FS layout report
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/wal
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/wal
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/data/data
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.913372 12263 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.917824 12263 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.919091 12263 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:30.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.930230 12263 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:06:30.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.943557 12263 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:06:30.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.944362 12263 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
07:06:30.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.944944 12263 ts_tablet_manager.cc:590] Registered 0 tablets
07:06:30.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:30.945385 12263 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:06:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.248760 12263 rpc_server.cc:307] RPC server started. Bound to: 127.11.179.129:43535
07:06:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.248936 12413 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.179.129:43535 every 8 connection(s)
07:06:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.253314 12263 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/data/info.pb
07:06:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.261737 11982 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 12263
07:06:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.262259 11982 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-0/wal/instance
07:06:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.276822 11982 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/wal
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/data
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/logs
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/data/info.pb
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.130:0
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.179.130
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.130
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:06:31.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.355847 12418 heartbeater.cc:347] Connected to a master server at 127.11.179.190:33769
07:06:31.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.356452 12418 heartbeater.cc:464] Registering TS with master...
07:06:31.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.357986 12418 heartbeater.cc:511] Master 127.11.179.190:33769 requested a full tablet report, sending...
07:06:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.360242 12415 heartbeater.cc:347] Connected to a master server at 127.11.179.189:34495
07:06:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.360667 12415 heartbeater.cc:464] Registering TS with master...
07:06:31.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.362197 12027 ts_manager.cc:194] Registered new tserver with Master: 858d42d1b4ca4d94935bbb0a87483127 (127.11.179.129:43535)
07:06:31.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.362663 12415 heartbeater.cc:511] Master 127.11.179.189:34495 requested a full tablet report, sending...
07:06:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.369625 12102 ts_manager.cc:194] Registered new tserver with Master: 858d42d1b4ca4d94935bbb0a87483127 (127.11.179.129:43535)
07:06:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.378805 12414 heartbeater.cc:347] Connected to a master server at 127.11.179.188:39405
07:06:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.379225 12414 heartbeater.cc:464] Registering TS with master...
07:06:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.380180 12414 heartbeater.cc:511] Master 127.11.179.188:39405 requested a full tablet report, sending...
07:06:31.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.384984 12181 ts_manager.cc:194] Registered new tserver with Master: 858d42d1b4ca4d94935bbb0a87483127 (127.11.179.129:43535)
07:06:31.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.390226 12181 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.179.129:58127
07:06:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:31.782866 12422 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:31.783401 12422 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:31.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:31.783859 12422 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:31.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:31.815102 12422 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:31.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:31.816092 12422 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.179.130
07:06:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.857048 12422 tablet_server_runner.cc:78] Tablet server non-default flags:
07:06:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/data
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/wal
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.130:0
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/data/info.pb
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.130
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.179.130
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/logs
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.858850 12422 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.861469 12422 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:31.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:31.885588 12431 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:32.396251 12414 heartbeater.cc:503] Master 127.11.179.188:39405 was elected leader, sending a full tablet report...
07:06:33.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:31.886399 12432 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:33.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:31.887569 12422 server_base.cc:1034] running on GCE node
07:06:33.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:31.889364 12434 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.045863 12422 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:33.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.048877 12422 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:33.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.050421 12422 hybrid_clock.cc:648] HybridClock initialized: now 1714115193050377 us; error 46 us; skew 500 ppm
07:06:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.051437 12422 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:33.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.058737 12422 webserver.cc:460] Webserver started at http://127.11.179.130:40725/ using document root <none> and password file <none>
07:06:33.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.059952 12422 fs_manager.cc:359] Metadata directory not provided
07:06:33.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.060243 12422 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:33.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.060798 12422 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.067484 12422 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/data/instance:
07:06:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4331bb516c0a4f34bf850f3fecefb210"
07:06:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:33 on dist-test-slave-9jsd"
07:06:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.069085 12422 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/wal/instance:
07:06:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4331bb516c0a4f34bf850f3fecefb210"
07:06:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:33 on dist-test-slave-9jsd"
07:06:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.078205 12422 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
07:06:33.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.084894 12443 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.086405 12422 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
07:06:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.086840 12422 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/wal
07:06:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4331bb516c0a4f34bf850f3fecefb210"
07:06:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:33 on dist-test-slave-9jsd"
07:06:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.087311 12422 fs_report.cc:383] FS layout report
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/wal
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/wal
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/data/data
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:33.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.159365 12422 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:33.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.160923 12422 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:33.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.161460 12422 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.164455 12422 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:06:33.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.169643 12422 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:06:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.169891 12422 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:06:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.170118 12422 ts_tablet_manager.cc:590] Registered 0 tablets
07:06:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.170269 12422 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:06:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.329707 12422 rpc_server.cc:307] RPC server started. Bound to: 127.11.179.130:40637
07:06:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.329819 12556 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.179.130:40637 every 8 connection(s)
07:06:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.332509 12422 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/data/info.pb
07:06:33.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.343580 11982 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 12422
07:06:33.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.344419 11982 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-1/wal/instance
07:06:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.366209 11982 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/wal
07:06:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/data
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/logs
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/data/info.pb
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.131:0
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.179.131
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.131
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:33.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:06:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.401537 12559 heartbeater.cc:347] Connected to a master server at 127.11.179.190:33769
07:06:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.402209 12559 heartbeater.cc:464] Registering TS with master...
07:06:33.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.403714 12559 heartbeater.cc:511] Master 127.11.179.190:33769 requested a full tablet report, sending...
07:06:33.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.404757 12558 heartbeater.cc:347] Connected to a master server at 127.11.179.189:34495
07:06:33.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.405140 12558 heartbeater.cc:464] Registering TS with master...
07:06:33.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.406311 12558 heartbeater.cc:511] Master 127.11.179.189:34495 requested a full tablet report, sending...
07:06:33.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.409281 12027 ts_manager.cc:194] Registered new tserver with Master: 4331bb516c0a4f34bf850f3fecefb210 (127.11.179.130:40637)
07:06:33.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.409518 12102 ts_manager.cc:194] Registered new tserver with Master: 4331bb516c0a4f34bf850f3fecefb210 (127.11.179.130:40637)
07:06:33.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.413560 12557 heartbeater.cc:347] Connected to a master server at 127.11.179.188:39405
07:06:33.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.414016 12557 heartbeater.cc:464] Registering TS with master...
07:06:33.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.415040 12557 heartbeater.cc:511] Master 127.11.179.188:39405 requested a full tablet report, sending...
07:06:33.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.417774 12181 ts_manager.cc:194] Registered new tserver with Master: 4331bb516c0a4f34bf850f3fecefb210 (127.11.179.130:40637)
07:06:33.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.419623 12181 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.179.130:52363
07:06:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:33.724063 12567 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:33.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:33.724694 12567 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:33.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:33.725450 12567 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:33.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:33.757493 12567 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:33.758495 12567 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.179.131
07:06:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.794348 12567 tablet_server_runner.cc:78] Tablet server non-default flags:
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.179.148:45713
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/data
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/wal
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.179.131:0
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/data/info.pb
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.179.131
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.179.190:33769,127.11.179.189:34495,127.11.179.188:39405
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.179.131
07:06:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/logs
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.796094 12567 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:33.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.798414 12567 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:33.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:33.817611 12574 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.423861 12557 heartbeater.cc:503] Master 127.11.179.188:39405 was elected leader, sending a full tablet report...
07:06:34.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:33.817929 12573 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:33.820140 12576 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:33.821225 12567 server_base.cc:1034] running on GCE node
07:06:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.980412 12567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.983134 12567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:34.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.984586 12567 hybrid_clock.cc:648] HybridClock initialized: now 1714115194984538 us; error 62 us; skew 500 ppm
07:06:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.985576 12567 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:34.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.993088 12567 webserver.cc:460] Webserver started at http://127.11.179.131:44071/ using document root <none> and password file <none>
07:06:34.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.994325 12567 fs_manager.cc:359] Metadata directory not provided
07:06:34.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.994637 12567 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:34.995203 12567 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.000792 12567 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/data/instance:
07:06:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47c457407bb94a9eb15841e2d82c5395"
07:06:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:34 on dist-test-slave-9jsd"
07:06:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.002493 12567 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/wal/instance:
07:06:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47c457407bb94a9eb15841e2d82c5395"
07:06:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:34 on dist-test-slave-9jsd"
07:06:35.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.012374 12567 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
07:06:35.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.019166 12584 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:35.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.020524 12567 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
07:06:35.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.020960 12567 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/wal
07:06:35.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47c457407bb94a9eb15841e2d82c5395"
07:06:35.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:34 on dist-test-slave-9jsd"
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.021489 12567 fs_report.cc:383] FS layout report
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/wal
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/wal
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/data/data
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.087332 12567 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.089090 12567 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:35.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.089763 12567 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.093238 12567 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:06:35.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.098790 12567 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:06:35.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.099057 12567 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:06:35.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.099380 12567 ts_tablet_manager.cc:590] Registered 0 tablets
07:06:35.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.099598 12567 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:06:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.254977 12567 rpc_server.cc:307] RPC server started. Bound to: 127.11.179.131:39865
07:06:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.255214 12698 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.179.131:39865 every 8 connection(s)
07:06:35.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.258095 12567 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/data/info.pb
07:06:35.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.264389 11982 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 12567
07:06:35.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.265012 11982 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster3649945498100158552/ts-2/wal/instance
07:06:35.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.304807 12699 heartbeater.cc:347] Connected to a master server at 127.11.179.188:39405
07:06:35.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.305316 12700 heartbeater.cc:347] Connected to a master server at 127.11.179.189:34495
07:06:35.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.305491 12699 heartbeater.cc:464] Registering TS with master...
07:06:35.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.305738 12700 heartbeater.cc:464] Registering TS with master...
07:06:35.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.306485 12701 heartbeater.cc:347] Connected to a master server at 127.11.179.190:33769
07:06:35.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.306906 12701 heartbeater.cc:464] Registering TS with master...
07:06:35.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.307297 12700 heartbeater.cc:511] Master 127.11.179.189:34495 requested a full tablet report, sending...
07:06:35.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.307291 12699 heartbeater.cc:511] Master 127.11.179.188:39405 requested a full tablet report, sending...
07:06:35.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.307897 12701 heartbeater.cc:511] Master 127.11.179.190:33769 requested a full tablet report, sending...
07:06:35.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.310451 12181 ts_manager.cc:194] Registered new tserver with Master: 47c457407bb94a9eb15841e2d82c5395 (127.11.179.131:39865)
07:06:35.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.311125 12027 ts_manager.cc:194] Registered new tserver with Master: 47c457407bb94a9eb15841e2d82c5395 (127.11.179.131:39865)
07:06:35.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.311234 12102 ts_manager.cc:194] Registered new tserver with Master: 47c457407bb94a9eb15841e2d82c5395 (127.11.179.131:39865)
07:06:35.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.313935 12181 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.179.131:34019
07:06:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:35.323716 11982 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:06:35.335 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:06:35.336 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:06:35.347 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.179.190"
port: 33769
}
bound_http_address {
host: "127.11.179.190"
port: 46339
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.179.189"
port: 34495
}
bound_http_address {
host: "127.11.179.189"
port: 43501
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.179.188"
port: 39405
}
bound_http_address {
host: "127.11.179.188"
port: 42889
}
}
}
07:06:35.353 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:06:35.356 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.11.179.129"
port: 43535
}
bound_http_address {
host: "127.11.179.129"
port: 41105
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.11.179.130"
port: 40637
}
bound_http_address {
host: "127.11.179.130"
port: 40725
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.11.179.131"
port: 39865
}
bound_http_address {
host: "127.11.179.131"
port: 44071
}
}
}
07:06:35.356 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:06:35.586 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:06:35.725 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLw1kUt/test-tmp
07:06:35.726 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:06:35.726 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLw1kUt/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:06:35.732 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:06:36.105 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:36.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:06:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:36.109300 12710 mini_oidc.cc:152] Starting JWKS server
07:06:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:36.113507 12710 webserver.cc:460] Webserver started at http://127.0.0.1:43793/ using document root <none> and password file <none>
07:06:36.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:36.114126 12710 mini_oidc.cc:189] Starting OIDC Discovery server
07:06:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:36.116158 12710 webserver.cc:460] Webserver started at http://127.0.0.1:40791/ using document root <none> and password file <none>
07:06:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:06:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/principal' for realm 'KRBTEST.COM',
07:06:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:06:36.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:36 dist-test-slave-9jsd krb5kdc[12723](info): setting up network...
07:06:36.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:06:36.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:36 dist-test-slave-9jsd krb5kdc[12723](info): set up 2 sockets
07:06:36.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:36 dist-test-slave-9jsd krb5kdc[12723](info): commencing operation
07:06:36.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:06:36.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:36.317657 12699 heartbeater.cc:503] Master 127.11.179.188:39405 was elected leader, sending a full tablet report...
07:06:37.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.205627 12710 mini_kdc.cc:121] Time spent starting KDC: real 1.045s user 0.005s sys 0.019s
07:06:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:06:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:06:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:06:37.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:06:37.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:06:37.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:06:37.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:06:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:06:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:06:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:06:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/test-user.keytab.
07:06:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/test-user.keytab.
07:06:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:37 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115197, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:06:37.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:06:37Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:06:37.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:06:37Z Disabled control of system clock
07:06:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.105.190@KRBTEST.COM; defaulting to no policy
07:06:37.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:06:37.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.105.190@KRBTEST.COM" created.
07:06:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:06:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.105.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.190.keytab.
07:06:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.105.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.190.keytab.
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:37.433813 12710 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/logs
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/info.pb
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.190:38807
07:06:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.190
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.190.keytab
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.190
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:37.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:06:37.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.773406 12736 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:37.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.774148 12736 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.774953 12736 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:37.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.806471 12736 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:06:37.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.806799 12736 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:37.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.807212 12736 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:37.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.807534 12736 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:37.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.807840 12736 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:37.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.808151 12736 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:37.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.808375 12736 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43793/jwks.json
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:37.846545 12736 master_runner.cc:386] Master server non-default flags:
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.190.keytab
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.190
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.190:38807
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:06:37.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/info.pb
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.190
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/logs
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:37.848218 12736 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:37.849665 12736 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.873629 12744 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.873692 12743 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:37.879060 12736 server_base.cc:1034] running on GCE node
07:06:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:37.875620 12746 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:39.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.050755 12736 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.054929 12736 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.056427 12736 hybrid_clock.cc:648] HybridClock initialized: now 1714115199056332 us; error 108 us; skew 500 ppm
07:06:39.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.057696 12736 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:39.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:39 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115199, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.066676 12736 init.cc:377] Logged in from keytab as kudu/127.12.105.190@KRBTEST.COM (short username kudu)
07:06:39.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.073002 12736 webserver.cc:460] Webserver started at http://127.12.105.190:44175/ using document root <none> and password file <none>
07:06:39.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.074417 12736 fs_manager.cc:359] Metadata directory not provided
07:06:39.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.074748 12736 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:39.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.075958 12736 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:39.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.082357 12736 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/instance:
07:06:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79ed869991de4062be52969542222320"
07:06:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:39 on dist-test-slave-9jsd"
07:06:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.083648 12736 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal/instance:
07:06:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79ed869991de4062be52969542222320"
07:06:39.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:39 on dist-test-slave-9jsd"
07:06:39.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.092548 12736 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
07:06:39.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.099943 12756 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.101576 12736 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.007s sys 0.000s
07:06:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.102125 12736 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79ed869991de4062be52969542222320"
07:06:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:39 on dist-test-slave-9jsd"
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.102669 12736 fs_report.cc:383] FS layout report
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/data
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.172008 12736 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.178774 12736 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.179255 12736 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:39.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.286176 12736 rpc_server.cc:307] RPC server started. Bound to: 127.12.105.190:38807
07:06:39.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.286288 12807 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.105.190:38807 every 8 connection(s)
07:06:39.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.289296 12736 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/info.pb
07:06:39.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.296339 12808 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:06:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.297358 12710 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 12736
07:06:39.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.297971 12710 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal/instance
07:06:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.314869 12808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } has no permanent_uuid. Determining permanent_uuid...
07:06:39.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.105.189@KRBTEST.COM; defaulting to no policy
07:06:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:06:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.105.189@KRBTEST.COM" created.
07:06:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:39 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115199, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.190@KRBTEST.COM for kudu/127.12.105.190@KRBTEST.COM
07:06:39.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:06:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.105.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.189.keytab.
07:06:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.105.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.189.keytab.
07:06:39.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.364274 12710 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/logs
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/info.pb
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.189:44215
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.189
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.189.keytab
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.189
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:06:39.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.375820 12808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } has no permanent_uuid. Determining permanent_uuid...
07:06:39.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.379281 12759 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.385830 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.189:44215: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.436504 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } attempt: 1
07:06:39.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.443706 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.189:44215: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:39.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.512257 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } attempt: 2
07:06:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.517246 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.189:44215: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:39.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.611325 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } attempt: 3
07:06:39.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.623183 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.189:44215: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:39.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.766702 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } attempt: 4
07:06:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.770913 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.189:44215: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:39.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.773676 12814 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:39.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.774273 12814 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.774888 12814 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.806864 12814 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:06:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.807118 12814 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.807384 12814 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.807588 12814 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.807785 12814 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.807991 12814 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.808137 12814 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43793/jwks.json
07:06:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.848253 12814 master_runner.cc:386] Master server non-default flags:
07:06:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data
07:06:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.189.keytab
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.189
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.189:44215
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/info.pb
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.189
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/logs
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.849576 12814 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:39.850941 12814 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:39.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.874626 12822 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:40.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:40.070397 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } attempt: 5
07:06:40.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:40.081939 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.189:44215: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:40.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:40.629568 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } attempt: 6
07:06:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:40.651449 12695 debug-util.cc:398] Leaking SignalData structure 0x7b08000be480 after lost signal to thread 12568
07:06:40.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:40.652609 12695 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0920 after lost signal to thread 12698
07:06:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:40.661645 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.189:44215: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.722218 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } attempt: 7
07:06:41.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:41.764356 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.189:44215: Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:41.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:41.273895 12820 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 12814
07:06:41.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:41.814059 12814 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.945s user 0.668s sys 1.076s
07:06:41.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:39.875789 12821 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:41.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:41.814862 12814 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.947s user 0.668s sys 1.077s
07:06:41.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:41.816107 12823 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1943 milliseconds
07:06:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:41.816872 12824 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.817113 12814 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:06:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.822014 12814 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:41.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.825356 12814 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:41.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.826970 12814 hybrid_clock.cc:648] HybridClock initialized: now 1714115201826933 us; error 65 us; skew 500 ppm
07:06:41.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.827867 12814 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:41.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:41 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115201, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.840046 12814 init.cc:377] Logged in from keytab as kudu/127.12.105.189@KRBTEST.COM (short username kudu)
07:06:41.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.847066 12814 webserver.cc:460] Webserver started at http://127.12.105.189:43211/ using document root <none> and password file <none>
07:06:41.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.848495 12814 fs_manager.cc:359] Metadata directory not provided
07:06:41.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.848876 12814 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:41.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.849572 12814 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.858072 12814 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/instance:
07:06:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e8116bdbc1c496f9f89820b63e10048"
07:06:41.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:41 on dist-test-slave-9jsd"
07:06:41.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.860045 12814 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal/instance:
07:06:41.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e8116bdbc1c496f9f89820b63e10048"
07:06:41.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:41 on dist-test-slave-9jsd"
07:06:41.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.872205 12814 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.011s sys 0.000s
07:06:41.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.881317 12838 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:41.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.882985 12814 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
07:06:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.883539 12814 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e8116bdbc1c496f9f89820b63e10048"
07:06:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:41 on dist-test-slave-9jsd"
07:06:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.884063 12814 fs_report.cc:383] FS layout report
07:06:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/data
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:41.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.969720 12814 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:41.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.976608 12814 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:41.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:41.977149 12814 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.094019 12814 rpc_server.cc:307] RPC server started. Bound to: 127.12.105.189:44215
07:06:42.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.094156 12891 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.105.189:44215 every 8 connection(s)
07:06:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.097795 12814 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/info.pb
07:06:42.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.102290 12710 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 12814
07:06:42.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.102871 12710 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal/instance
07:06:42.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.107635 12892 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:06:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.105.188@KRBTEST.COM; defaulting to no policy
07:06:42.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:06:42.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.105.188@KRBTEST.COM" created.
07:06:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.133487 12892 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } has no permanent_uuid. Determining permanent_uuid...
07:06:42.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:42 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115201, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.189@KRBTEST.COM for kudu/127.12.105.190@KRBTEST.COM
07:06:42.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:06:42.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.105.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.188.keytab.
07:06:42.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.105.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.188.keytab.
07:06:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.171653 12710 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data
07:06:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/logs
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/info.pb
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.188:40859
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.188
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.188.keytab
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.188
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:06:42.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:06:42.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.184963 12892 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } has no permanent_uuid. Determining permanent_uuid...
07:06:42.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:42 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115201, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.189@KRBTEST.COM for kudu/127.12.105.189@KRBTEST.COM
07:06:42.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.220649 12892 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } has no permanent_uuid. Determining permanent_uuid...
07:06:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.224206 12840 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.233225 12892 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:42.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.282948 12892 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 1
07:06:42.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.290150 12892 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:42.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.359462 12892 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 2
07:06:42.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.366817 12892 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.458295 12892 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 3
07:06:42.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.466830 12892 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.545276 12212 debug-util.cc:398] Leaking SignalData structure 0x7b08000b20c0 after lost signal to thread 12144
07:06:42.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.546093 12212 debug-util.cc:398] Leaking SignalData structure 0x7b08000a00e0 after lost signal to thread 12215
07:06:42.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.595748 12899 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:42.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.596345 12899 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:42.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.597477 12899 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:42.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.610363 12892 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 4
07:06:42.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.617326 12892 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:42.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.665612 12899 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:06:42.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.666383 12899 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:42.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.667194 12899 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.667785 12899 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.668311 12899 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.668874 12899 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.669322 12899 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43793/jwks.json
07:06:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.748610 12899 master_runner.cc:386] Master server non-default flags:
07:06:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data
07:06:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.188.keytab
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.188
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.188:40859
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/info.pb
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.188
07:06:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/logs
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:42.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.750458 12899 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.751996 12899 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:42.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.776830 12909 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:42.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:42.916821 12892 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 5
07:06:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.924230 12892 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:43.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:43.471747 12892 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 6
07:06:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:43.508981 12892 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:43.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:43.854974 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } attempt: 8
07:06:43.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:43 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115199, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.190@KRBTEST.COM for kudu/127.12.105.189@KRBTEST.COM
07:06:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:43.953573 12808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } has no permanent_uuid. Determining permanent_uuid...
07:06:43.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:43.962522 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.028213 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 1
07:06:44.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.036420 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.090067 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 2
07:06:44.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.096371 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.172838 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 3
07:06:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.186782 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.342481 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 4
07:06:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.352236 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.569515 12892 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 7
07:06:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.609287 12892 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:44.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.648942 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 5
07:06:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.652540 12758 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111) [suppressed 12 similar messages]
07:06:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.656618 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:44.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.172679 12908 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 12899
07:06:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.961215 12899 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.182s user 0.527s sys 1.016s
07:06:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:42.779718 12910 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.961653 12899 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.183s user 0.527s sys 1.016s
07:06:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.962955 12912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:44.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:44.967530 12911 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2181 milliseconds
07:06:44.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.967576 12899 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:06:44.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.969486 12899 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:44.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.976198 12899 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.977816 12899 hybrid_clock.cc:648] HybridClock initialized: now 1714115204977642 us; error 193 us; skew 500 ppm
07:06:44.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.978852 12899 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:44 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115204, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:44.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.985546 12899 init.cc:377] Logged in from keytab as kudu/127.12.105.188@KRBTEST.COM (short username kudu)
07:06:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.992159 12899 webserver.cc:460] Webserver started at http://127.12.105.188:43797/ using document root <none> and password file <none>
07:06:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.993209 12899 fs_manager.cc:359] Metadata directory not provided
07:06:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.993530 12899 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.994021 12899 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:06:45.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:44.999390 12899 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/instance:
07:06:45.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17f0a2bc7dcf45d7a0816f6824f138f3"
07:06:45.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:44 on dist-test-slave-9jsd"
07:06:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.000706 12899 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal/instance:
07:06:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17f0a2bc7dcf45d7a0816f6824f138f3"
07:06:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:44 on dist-test-slave-9jsd"
07:06:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.010903 12899 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
07:06:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.019161 12930 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.020738 12899 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.004s
07:06:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.021153 12899 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17f0a2bc7dcf45d7a0816f6824f138f3"
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:44 on dist-test-slave-9jsd"
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.021561 12899 fs_report.cc:383] FS layout report
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/data
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:45.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.089483 12899 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.098978 12899 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.099671 12899 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:45.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.178365 12808 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } attempt: 6
07:06:45.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:45.189435 12808 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.12.105.188:40859: Network error: Client connection negotiation failed: client connection to 127.12.105.188:40859: connect: Connection refused (error 111)
07:06:45.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.230257 12899 rpc_server.cc:307] RPC server started. Bound to: 127.12.105.188:40859
07:06:45.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.230367 12986 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.105.188:40859 every 8 connection(s)
07:06:45.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.233377 12899 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/info.pb
07:06:45.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.239563 12710 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 12899
07:06:45.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.240152 12710 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal/instance
07:06:45.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.243455 12987 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:06:45.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.272392 12987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } has no permanent_uuid. Determining permanent_uuid...
07:06:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:45 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115197, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.105.188@KRBTEST.COM
07:06:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:45 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115204, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.188@KRBTEST.COM for kudu/127.12.105.190@KRBTEST.COM
07:06:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.345760 12987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } has no permanent_uuid. Determining permanent_uuid...
07:06:45.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:45 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115197, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.105.189@KRBTEST.COM
07:06:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:45 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115204, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.188@KRBTEST.COM for kudu/127.12.105.189@KRBTEST.COM
07:06:45.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.383731 12987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } has no permanent_uuid. Determining permanent_uuid...
07:06:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:45 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115197, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.105.190@KRBTEST.COM
07:06:45.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:45 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115204, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.188@KRBTEST.COM for kudu/127.12.105.188@KRBTEST.COM
07:06:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.422863 12710 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:06:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.424971 12987 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Bootstrap starting.
07:06:45.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:45.432 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:06:45.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.434939 12987 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Neither blocks nor log segments found. Creating new log.
07:06:45.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.438107 12987 log.cc:824] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Log is configured to *not* fsync() on all Append() calls
07:06:45.444 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:06:45.445 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:06:45.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.105.190"
port: 38807
}
bound_http_address {
host: "127.12.105.190"
port: 44175
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.105.189"
port: 44215
}
bound_http_address {
host: "127.12.105.189"
port: 43211
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.105.188"
port: 40859
}
bound_http_address {
host: "127.12.105.188"
port: 43797
}
}
}
07:06:45.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:06:45.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:45.451 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:06:45.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:06:45.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.464083 12987 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: No bootstrap required, opened a new log
07:06:45.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:45 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115205, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:45.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.507105 12987 raft_consensus.cc:348] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.508188 12987 raft_consensus.cc:374] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.508522 12987 raft_consensus.cc:729] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 17f0a2bc7dcf45d7a0816f6824f138f3, State: Initialized, Role: FOLLOWER
07:06:45.510 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.509871 12987 consensus_queue.cc:260] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:06:45.512 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc
07:06:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.513545 12998 sys_catalog.cc:455] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } } }
07:06:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.514827 12998 sys_catalog.cc:458] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [sys.catalog]: This master's current role is: FOLLOWER
07:06:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.518916 12987 sys_catalog.cc:564] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [sys.catalog]: configured and running, proceeding with master startup.
07:06:45.520 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:06:45.521 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc
07:06:45.523 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:06:45.529 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:06:45.532 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:06:45.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:06:45.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:45.580063 13011 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:45.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:45.581351 13011 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:45.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:45 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115205, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:45.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:06:45.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:45.600 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc
07:06:45.602 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:06:45.602 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.105.189:44215
07:06:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.563887 12899 master_runner.cc:186] Error getting master registration for 127.12.105.190:38807: OK, instance_id { permanent_uuid: "79ed869991de4062be52969542222320" instance_seqno: 1714115199249468 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:06:45.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.604281 12899 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:06:45.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:45.608821 12936 tablet.cc:2369] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:06:45.614 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:06:45.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.615056 12710 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 12814
07:06:45.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:45.649 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.105.188:40859
07:06:45.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:06:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.652110 12710 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 12899
07:06:45.682 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:45.682 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.105.190:38807
07:06:45.682 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:06:45.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.683138 12710 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 12736
07:06:45.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:45.708 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.105.189:44215
07:06:45.711 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:06:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:45.713089 12710 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data
07:06:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:45.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/logs
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/info.pb
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.189:44215
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.189
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43211
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.189.keytab
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.189
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:06:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.055547 13014 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.056139 13014 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.056736 13014 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.091384 13014 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:06:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.091650 13014 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.091921 13014 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.092128 13014 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.092343 13014 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.092533 13014 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.092680 13014 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43793/jwks.json
07:06:46.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:46.131448 13014 master_runner.cc:386] Master server non-default flags:
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.189.keytab
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.189
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.189:44215
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/info.pb
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.189
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43211
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/logs
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:46.132717 13014 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:46.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:46.133864 13014 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:46.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.154052 13022 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:47.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.154083 13026 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:47.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:46.155050 13023 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:47.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:46.155853 13014 server_base.cc:1034] running on GCE node
07:06:47.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.331465 13014 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:47.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.335187 13014 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:47.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.338464 13014 hybrid_clock.cc:648] HybridClock initialized: now 1714115207338424 us; error 48 us; skew 500 ppm
07:06:47.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.339426 13014 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:47 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115207, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.351763 13014 init.cc:377] Logged in from keytab as kudu/127.12.105.189@KRBTEST.COM (short username kudu)
07:06:47.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.357582 13014 webserver.cc:460] Webserver started at http://127.12.105.189:43211/ using document root <none> and password file <none>
07:06:47.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.358758 13014 fs_manager.cc:359] Metadata directory not provided
07:06:47.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.359015 13014 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:47.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.368491 13014 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
07:06:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.374051 13034 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:47.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.375396 13014 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
07:06:47.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.375829 13014 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:47.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e8116bdbc1c496f9f89820b63e10048"
07:06:47.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:41 on dist-test-slave-9jsd"
07:06:47.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.378473 13014 fs_report.cc:383] FS layout report
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/data
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:47.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.449563 13014 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:47.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.455785 13014 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:47.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.456269 13014 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.541234 13014 rpc_server.cc:307] RPC server started. Bound to: 127.12.105.189:44215
07:06:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.541358 13086 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.105.189:44215 every 8 connection(s)
07:06:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.544571 13014 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/data/info.pb
07:06:47.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.545883 12710 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 13014
07:06:47.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:47.546 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.105.188:40859
07:06:47.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.548720 12710 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/logs
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/info.pb
07:06:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:47.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:47.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:47.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:47.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.188:40859
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.188
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43797
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.188.keytab
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.188
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:06:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.564018 13087 sys_catalog.cc:263] Verifying existing consensus state
07:06:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240426 07:06:47.565197 13087 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:06:47.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.565809 13014 master.cc:537] Master@127.12.105.189:44215 shutting down...
07:06:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.634151 13014 master.cc:552] Master@127.12.105.189:44215 shutdown complete.
07:06:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:06:47.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.914295 13089 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.914954 13089 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.915620 13089 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:47.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.947423 13089 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:06:47.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.947748 13089 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:47.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.948194 13089 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:47.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.948554 13089 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:47.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.948846 13089 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:47.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.949148 13089 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:47.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:47.949414 13089 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43793/jwks.json
07:06:47.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.988168 13089 master_runner.cc:386] Master server non-default flags:
07:06:47.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:47.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:47.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:47.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:47.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data
07:06:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:06:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.188.keytab
07:06:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.188
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.188:40859
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/info.pb
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.188
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43797
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/logs
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.989853 13089 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:47.991432 13089 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:48.010527 13095 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:49.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:48.010816 13096 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:49.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:48.011845 13098 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:49.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:48.012001 13089 server_base.cc:1034] running on GCE node
07:06:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.174849 13089 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:49.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.178050 13089 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:49.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.179508 13089 hybrid_clock.cc:648] HybridClock initialized: now 1714115209179477 us; error 40 us; skew 500 ppm
07:06:49.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.180459 13089 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:49.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:49 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115209, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:49.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.187340 13089 init.cc:377] Logged in from keytab as kudu/127.12.105.188@KRBTEST.COM (short username kudu)
07:06:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.193619 13089 webserver.cc:460] Webserver started at http://127.12.105.188:43797/ using document root <none> and password file <none>
07:06:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.195120 13089 fs_manager.cc:359] Metadata directory not provided
07:06:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.195425 13089 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:49.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.204341 13089 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.003s sys 0.003s
07:06:49.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.209995 13108 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.211370 13089 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
07:06:49.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.211841 13089 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:49.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17f0a2bc7dcf45d7a0816f6824f138f3"
07:06:49.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:44 on dist-test-slave-9jsd"
07:06:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.214489 13089 fs_report.cc:383] FS layout report
07:06:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/wal
07:06:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/data
07:06:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.270895 13089 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:49.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.277761 13089 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:49.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.278307 13089 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.361305 13089 rpc_server.cc:307] RPC server started. Bound to: 127.12.105.188:40859
07:06:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.361460 13159 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.105.188:40859 every 8 connection(s)
07:06:49.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.364537 13089 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-2/data/info.pb
07:06:49.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.369326 12710 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 13089
07:06:49.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:49.371 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.105.190:38807
07:06:49.371 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:06:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.373091 12710 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu
07:06:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data
07:06:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:06:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/logs
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/info.pb
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.190:38807
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.190
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44175
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.190.keytab
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.190
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:06:49.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.380566 13160 sys_catalog.cc:263] Verifying existing consensus state
07:06:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.386586 13160 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Bootstrap starting.
07:06:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.429494 13160 log.cc:824] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Log is configured to *not* fsync() on all Append() calls
07:06:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.435385 13160 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:06:49.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.436483 13160 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Bootstrap complete.
07:06:49.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.474893 13160 raft_consensus.cc:348] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:06:49.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.477733 13160 raft_consensus.cc:374] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:06:49.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.478021 13160 raft_consensus.cc:729] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 17f0a2bc7dcf45d7a0816f6824f138f3, State: Initialized, Role: FOLLOWER
07:06:49.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.478937 13160 consensus_queue.cc:260] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:06:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.482159 13164 sys_catalog.cc:455] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } } }
07:06:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.483175 13164 sys_catalog.cc:458] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [sys.catalog]: This master's current role is: FOLLOWER
07:06:49.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.484051 13160 sys_catalog.cc:564] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [sys.catalog]: configured and running, proceeding with master startup.
07:06:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.529893 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.530257 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:49.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.533962 13114 tablet.cc:2369] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:06:49.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.529235 13089 master_runner.cc:186] Error getting master registration for 127.12.105.190:38807: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111),
07:06:49.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.552505 13089 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:06:49.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.715145 13164 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:06:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.715821 13164 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:06:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.721546 13111 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111)
07:06:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.723330 13164 leader_election.cc:288] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79ed869991de4062be52969542222320 (127.12.105.190:38807), 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215)
07:06:49.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.739190 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79ed869991de4062be52969542222320 (127.12.105.190:38807): Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111)
07:06:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.739971 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215): Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.740401 13111 leader_election.cc:302] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17f0a2bc7dcf45d7a0816f6824f138f3; no voters: 5e8116bdbc1c496f9f89820b63e10048, 79ed869991de4062be52969542222320
07:06:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.741367 13164 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:06:49.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.812222 13162 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:06:49.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.812976 13162 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:06:49.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.813758 13162 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:06:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.846961 13162 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:06:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.847280 13162 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:06:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.847692 13162 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:06:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.848006 13162 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:06:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.848301 13162 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:06:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.848600 13162 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:06:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.848845 13162 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43793/jwks.json
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.906774 13162 master_runner.cc:386] Master server non-default flags:
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.105.148:35501
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.105.190:38807,127.12.105.189:44215,127.12.105.188:40859
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/krb5kdc/kudu_127.12.105.190.keytab
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.105.190
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.105.190:38807
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:06:49.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43793/jwks.json
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/info.pb
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.105.190
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44175
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/logs
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 332b9f3d8b5b82a7ffa0661d384e58774172b5d3
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 06:24:53 UTC on 1a15035a6830
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 252
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:06:49.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.908536 13162 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:06:49.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.910118 13162 file_cache.cc:492] Constructed file cache file cache with capacity 419430
07:06:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.934294 13186 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:50.534801 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:50.535166 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:49.935662 13162 server_base.cc:1034] running on GCE node
07:06:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.937011 13188 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:49.934497 13185 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:06:51.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.155784 13162 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:06:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.159401 13162 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:06:51.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.160959 13162 hybrid_clock.cc:648] HybridClock initialized: now 1714115211160921 us; error 52 us; skew 500 ppm
07:06:51.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.161918 13162 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:06:51.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:51 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115211, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:51.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.170439 13162 init.cc:377] Logged in from keytab as kudu/127.12.105.190@KRBTEST.COM (short username kudu)
07:06:51.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.176627 13162 webserver.cc:460] Webserver started at http://127.12.105.190:44175/ using document root <none> and password file <none>
07:06:51.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.177726 13162 fs_manager.cc:359] Metadata directory not provided
07:06:51.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.178014 13162 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:06:51.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.187114 13162 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.004s
07:06:51.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.192082 13196 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:06:51.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.193611 13162 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
07:06:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.194042 13162 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data,/tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79ed869991de4062be52969542222320"
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 07:06:39 on dist-test-slave-9jsd"
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.196801 13162 fs_report.cc:383] FS layout report
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/data
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:06:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:06:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:06:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:06:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:06:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:06:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.251704 13162 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
07:06:51.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.261086 13162 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:06:51.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.261860 13162 kserver.cc:163] Server-wide thread pool size limit: 3276
07:06:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.300494 13214 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:06:51.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.301398 13214 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:06:51.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.310715 13214 leader_election.cc:288] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79ed869991de4062be52969542222320 (127.12.105.190:38807), 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215)
07:06:51.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:51.327596 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215): Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:51.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:51.328442 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79ed869991de4062be52969542222320 (127.12.105.190:38807): Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111)
07:06:51.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.328919 13111 leader_election.cc:302] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17f0a2bc7dcf45d7a0816f6824f138f3; no voters: 5e8116bdbc1c496f9f89820b63e10048, 79ed869991de4062be52969542222320
07:06:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.330087 13214 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:06:51.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.375144 13162 rpc_server.cc:307] RPC server started. Bound to: 127.12.105.190:38807
07:06:51.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.375924 13251 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.105.190:38807 every 8 connection(s)
07:06:51.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.379536 13162 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/data/info.pb
07:06:51.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.390452 12710 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu as pid 13162
07:06:51.391 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:06:51.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.394702 13252 sys_catalog.cc:263] Verifying existing consensus state
07:06:51.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240426 07:06:51.395875 13252 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:06:51.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.396418 13162 master.cc:537] Master@127.12.105.190:38807 shutting down...
07:06:51.454 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:51.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:51.481503 13162 master.cc:552] Master@127.12.105.190:38807 shutdown complete.
07:06:51.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskLw1kUt/test-tmp/mini-kudu-cluster8316033435767121729/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:06:51.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:51.540305 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:51.541838 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:51.547 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.105.188:40859(127.12.105.188:40859)] connecting to peer
07:06:51.548 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:51.640 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.105.188:40859(127.12.105.188:40859)] Successfully connected to peer
07:06:51.647 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:51.647 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:51.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:51.664 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:51.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:51.666 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:51.736 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:06:51.746 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.12.105.188
07:06:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:51 dist-test-slave-9jsd krb5kdc[12723](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115205, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.105.188@KRBTEST.COM
07:06:51.987 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf1adceb7, L:/127.0.0.1:38128 - R:/127.12.105.188:40859] using SASL/GSSAPI
07:06:52.005 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.019 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.024 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.030 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.030 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.031 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.038 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.040 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.045 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.045 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.046 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.046 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.047 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.047 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.058 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.060 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.061 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.062 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.063 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.063 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.063 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.064 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.065 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.078 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.080 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.083 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.084 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.086 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.098 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.099 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.102 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.102 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.103 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.103 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.103 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.104 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.106 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.178 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.180 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.182 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.182 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.183 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.183 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.185 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.217 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.220 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.222 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.223 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.223 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.223 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.224 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.224 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.226 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.258 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.260 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.262 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.263 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.263 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.263 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.265 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.417 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:52.420 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:52.421 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.421 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.422 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:52.424 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.424 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.427 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:52.428 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:52.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:52.545611 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:52.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:52.545929 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:53.197 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:53.199 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:53.201 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:53.201 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:53.201 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:53.201 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:53.201 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:53.201 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:53.203 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:53.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:53.548889 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:53.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:53.549224 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:54.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:54.262984 13264 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:06:54.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:54.263387 13264 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:06:54.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:54.265130 13264 leader_election.cc:288] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79ed869991de4062be52969542222320 (127.12.105.190:38807), 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215)
07:06:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:54.272104 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79ed869991de4062be52969542222320 (127.12.105.190:38807): Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111)
07:06:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:54.272879 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215): Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:54.273387 13111 leader_election.cc:302] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17f0a2bc7dcf45d7a0816f6824f138f3; no voters: 5e8116bdbc1c496f9f89820b63e10048, 79ed869991de4062be52969542222320
07:06:54.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:54.274268 13264 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:06:54.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:54.552397 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:54.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:54.552742 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:54.554996 13089 master_runner.cc:186] Error getting master registration for 127.12.105.190:38807: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111),
07:06:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:54.555402 13089 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:06:54.617 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:54.619 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:54.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:54.621 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:54.622 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:54.623 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:54.623 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:54.624 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:54.624 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:55.556118 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:55.556423 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:55.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:55.986689 13269 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:06:55.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:55.987156 13269 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:06:55.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:55.988571 13269 leader_election.cc:288] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79ed869991de4062be52969542222320 (127.12.105.190:38807), 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215)
07:06:55.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:55.991466 13111 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:06:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:55.994899 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79ed869991de4062be52969542222320 (127.12.105.190:38807): Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111)
07:06:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:55.995409 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215): Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:06:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:55.995666 13111 leader_election.cc:302] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17f0a2bc7dcf45d7a0816f6824f138f3; no voters: 5e8116bdbc1c496f9f89820b63e10048, 79ed869991de4062be52969542222320
07:06:55.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:55.996238 13269 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:06:56.078 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:56.079 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:56.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:56.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:56.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:56.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:56.083 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:56.084 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:56.084 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:56.298 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:56.300 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:56.302 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:56.302 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:56.302 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:56.302 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:56.302 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:56.303 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:56.304 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:56.559475 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:56.559770 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:57.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:57.563409 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:57.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:57.563709 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:58.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:58.568125 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:58.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:58.568501 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:06:59.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:06:59 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115219, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:06:59.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:59.191511 13104 init.cc:295] Successfully reacquired a new kerberos TGT
07:06:59.238 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:06:59.240 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:06:59.242 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:59.242 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:59.242 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:59.242 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:59.242 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:06:59.242 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:06:59.245 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:06:59.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:59.559504 13089 master_runner.cc:186] Error getting master registration for 127.12.105.190:38807: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111),
07:06:59.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:06:59.560111 13089 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:06:59.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:59.571568 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:06:59.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:06:59.571887 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:00.057 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:00.059 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:00.060 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:00.061 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:00.061 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:00.062 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:00.062 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:00.062 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:00.063 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:00.574867 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:00.575165 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:01.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:01.312213 13279 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:07:01.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:01.312752 13279 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:07:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:01.314504 13279 leader_election.cc:288] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79ed869991de4062be52969542222320 (127.12.105.190:38807), 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215)
07:07:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:01.317800 13111 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111) [suppressed 1 similar messages]
07:07:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:01.321727 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79ed869991de4062be52969542222320 (127.12.105.190:38807): Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111)
07:07:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:01.322445 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215): Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:07:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:01.322800 13111 leader_election.cc:302] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17f0a2bc7dcf45d7a0816f6824f138f3; no voters: 5e8116bdbc1c496f9f89820b63e10048, 79ed869991de4062be52969542222320
07:07:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:01.323570 13279 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:07:01.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:01.578384 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:01.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:01.578676 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:01.778 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:01.779 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:01.783 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:01.783 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:01.783 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:01.783 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:01.783 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:01.784 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:01.784 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:02.198 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:02.199 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:02.201 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:02.201 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:02.202 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:02.205 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:02.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:02.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:02.206 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:02.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:02.581631 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:02.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:02.581915 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:02.878 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:02.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:02.880 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:02.880 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:02.880 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:02.883 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:02.883 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:02.883 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:02.887 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:03.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:03.584905 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:03.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:03.585234 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:04.563421 13089 master_runner.cc:186] Error getting master registration for 127.12.105.190:38807: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111),
07:07:04.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:04.563871 13089 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:07:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:04.589668 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:04.590063 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:04.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:04.706593 13289 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:07:04.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:04.707051 13289 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "79ed869991de4062be52969542222320" member_type: VOTER last_known_addr { host: "127.12.105.190" port: 38807 } } peers { permanent_uuid: "5e8116bdbc1c496f9f89820b63e10048" member_type: VOTER last_known_addr { host: "127.12.105.189" port: 44215 } } peers { permanent_uuid: "17f0a2bc7dcf45d7a0816f6824f138f3" member_type: VOTER last_known_addr { host: "127.12.105.188" port: 40859 } }
07:07:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:04.708515 13289 leader_election.cc:288] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 79ed869991de4062be52969542222320 (127.12.105.190:38807), 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215)
07:07:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:04.713670 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 79ed869991de4062be52969542222320 (127.12.105.190:38807): Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111)
07:07:04.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:04.714311 13111 leader_election.cc:334] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5e8116bdbc1c496f9f89820b63e10048 (127.12.105.189:44215): Network error: Client connection negotiation failed: client connection to 127.12.105.189:44215: connect: Connection refused (error 111)
07:07:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:04.714654 13111 leader_election.cc:302] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17f0a2bc7dcf45d7a0816f6824f138f3; no voters: 5e8116bdbc1c496f9f89820b63e10048, 79ed869991de4062be52969542222320
07:07:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:04.715346 13289 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:07:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:05.593113 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:05.593446 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:06.298 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:06.300 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:06.301 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:06.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:06.302 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:06.302 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:06.302 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:06.302 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:06.304 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:06.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:06.596899 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:06.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:06.597254 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:07.457 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:07.458 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:07.459 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:07.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:07.460 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:07.460 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:07.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:07.460 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:07.462 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:07.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:07.600441 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:07.600744 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:08.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:08.604696 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:08.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:08.605058 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:07:09 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115229, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:07:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:09.194694 13104 init.cc:295] Successfully reacquired a new kerberos TGT
07:07:09.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:09.567996 13089 master_runner.cc:186] Error getting master registration for 127.12.105.190:38807: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111),
07:07:09.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:09.568456 13089 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:07:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:09.608055 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:09.608343 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:09.737 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:09.738 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:09.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:09.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:09.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:09.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:09.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:09.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:09.743 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:10.611529 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:10.611847 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:10.617 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:10.618 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:10.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:10.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:10.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:10.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:10.620 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:10.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:10.622 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:10.681787 12133 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1e80 after lost signal to thread 12068
07:07:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:10.682391 12133 debug-util.cc:398] Leaking SignalData structure 0x7b08000a2900 after lost signal to thread 12136
07:07:11.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:11.614945 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:11.615236 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:12.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:12.618366 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:12.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:12.618690 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:12.938 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:12.939 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:12.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:12.940 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:12.940 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:12.940 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:12.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:12.941 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:12.943 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:13.621785 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:13.622085 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:14.278 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:14.279 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:14.280 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:14.280 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:14.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:14.281 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:14.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:14.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:14.283 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:14.572075 13089 master_runner.cc:186] Error getting master registration for 127.12.105.190:38807: Network error: Client connection negotiation failed: client connection to 127.12.105.190:38807: connect: Connection refused (error 111),
07:07:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:14.572564 13089 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:07:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:14.625345 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:14.625666 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:14.737 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:14.738 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:14.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:14.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:14.740 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:14.740 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:14.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:14.740 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:14.742 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:15.628703 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:15.629047 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:16.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:16.632083 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:16.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:16.632395 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:16.658 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:16.659 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:16.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:16.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:16.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:16.660 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:16.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:16.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:16.663 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:17.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:17.635416 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:17.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:17.635718 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:18.639442 13175 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17f0a2bc7dcf45d7a0816f6824f138f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:07:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 07:07:18.639840 13175 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:07:19.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 07:07:19 dist-test-slave-9jsd krb5kdc[12723](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714115239, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.105.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:07:19.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:19.197396 13104 init.cc:295] Successfully reacquired a new kerberos TGT
07:07:19.377 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.189:44215(127.12.105.189:44215)] connecting to peer
07:07:19.379 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.12.105.190:38807(127.12.105.190:38807)] connecting to peer
07:07:19.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:19.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.189:44215(127.12.105.189:44215)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:19.380 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:19.380 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.189:44215: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215
07:07:19.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.105.190:38807(127.12.105.190:38807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:19.380 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.105.190:38807: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
07:07:19.383 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807; will retry
07:07:19.388 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=27987), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [246ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [263ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [599ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [604ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [627ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [628ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [630ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [643ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [643ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [645ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [655ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [660ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [661ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [662ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [680ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [682ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [682ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [700ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [702ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [704ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [780ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [780ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [782ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [821ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [822ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [823ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [855ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [859ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [861ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [862ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [863ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1015ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1026ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1795ms] refreshing cache from master, [1795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [1796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [1797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [1799ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1799ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [1801ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [1802ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [3215ms] refreshing cache from master, [3215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [3216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [3217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [3220ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [3221ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [3222ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [3223ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [4675ms] refreshing cache from master, [4675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [4679ms] too many traces: truncated at 100 traces, deferred=Deferred@25312984(state=PENDING, result=null, callback=(continuation of Deferred@2080100185 after retry RPC after error@691316415) -> (continuation of Deferred@296106798 after retry RPC after error@21144380) -> (continuation of Deferred@619062602 after retry RPC after error@678133303) -> (continuation of Deferred@146038623 after retry RPC after error@486229558) -> (continuation of Deferred@153287830 after retry RPC after error@2051776835) -> (continuation of Deferred@1353668594 after retry RPC after error@379805991) -> (continuation of Deferred@142342441 after retry RPC after error@798822620) -> (continuation of Deferred@94880282 after retry RPC after error@142619088) -> (continuation of Deferred@1600463704 after retry RPC after error@414831512) -> (continuation of Deferred@590706055 after retry RPC after error@716361344) -> (continuation of Deferred@1547388781 after retry RPC after error@180910689) -> (continuation of Deferred@47824882 after retry RPC after error@1042528231) -> (continuation of Deferred@950378337 after retry RPC after error@1903681894) -> (continuation of Deferred@1050601109 after retry RPC after error@488759854) -> (continuation of Deferred@166410364 after retry RPC after error@1197053174) -> (continuation of Deferred@247097542 after retry RPC after error@471391149) -> (continuation of Deferred@1281370250 after retry RPC after error@1201011668) -> (continuation of Deferred@439637944 after retry RPC after error@1076568800) -> (continuation of Deferred@222455319 after retry RPC after error@1709270832) -> (continuation of Deferred@1573942691 after retry RPC after error@1148547291) -> (continuation of Deferred@1520030876 after retry RPC after error@875556888) -> (continuation of Deferred@1779234387 after retry RPC after error@84979809) -> (continuation of Deferred@288995691 after retry RPC after error@1516053604) -> (continuation of Deferred@1466887954 after retry RPC after error@1467862566) -> (continuation of Deferred@190039875 after retry RPC after error@941538377) -> (continuation of Deferred@405328423 after retry RPC after error@555557882) -> (continuation of Deferred@887198021 after retry RPC after error@2043711332), errback=(continuation of Deferred@2080100185 after retry RPC after error@691316415) -> (continuation of Deferred@296106798 after retry RPC after error@21144380) -> (continuation of Deferred@619062602 after retry RPC after error@678133303) -> (continuation of Deferred@146038623 after retry RPC after error@486229558) -> (continuation of Deferred@153287830 after retry RPC after error@2051776835) -> (continuation of Deferred@1353668594 after retry RPC after error@379805991) -> (continuation of Deferred@142342441 after retry RPC after error@798822620) -> (continuation of Deferred@94880282 after retry RPC after error@142619088) -> (continuation of Deferred@1600463704 after retry RPC after error@414831512) -> (continuation of Deferred@590706055 after retry RPC after error@716361344) -> (continuation of Deferred@1547388781 after retry RPC after error@180910689) -> (continuation of Deferred@47824882 after retry RPC after error@1042528231) -> (continuation of Deferred@950378337 after retry RPC after error@1903681894) -> (continuation of Deferred@1050601109 after retry RPC after error@488759854) -> (continuation of Deferred@166410364 after retry RPC after error@1197053174) -> (continuation of Deferred@247097542 after retry RPC after error@471391149) -> (continuation of Deferred@1281370250 after retry RPC after error@1201011668) -> (continuation of Deferred@439637944 after retry RPC after error@1076568800) -> (continuation of Deferred@222455319 after retry RPC after error@1709270832) -> (continuation of Deferred@1573942691 after retry RPC after error@1148547291) -> (continuation of Deferred@1520030876 after retry RPC after error@875556888) -> (continuation of Deferred@1779234387 after retry RPC after error@84979809) -> (continuation of Deferred@288995691 after retry RPC after error@1516053604) -> (continuation of Deferred@1466887954 after retry RPC after error@1467862566) -> (continuation of Deferred@190039875 after retry RPC after error@941538377) -> (continuation of Deferred@405328423 after retry RPC after error@555557882) -> (continuation of Deferred@887198021 after retry RPC after error@2043711332)))
07:07:19.394 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.105.190:38807(127.12.105.190:38807)] disconnecting while in state TERMINATED
07:07:19.395 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.105.189:44215(127.12.105.189:44215)] disconnecting while in state TERMINATED
07:07:19.396 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.105.188:40859(127.12.105.188:40859)] disconnecting while in state READY
07:07:19.402 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:07:19.404 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.105.188:40859(127.12.105.188:40859)] handling channelInactive
07:07:19.404 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.105.188:40859(127.12.105.188:40859)] cleaning up while in state READY due to: connection closed
07:07:19.407 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:07:19.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:19.410255 12710 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 13089
07:07:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:07:20Z chronyd exiting
07:07:21.573 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:07:21.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:21.575848 11982 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 12263
07:07:21.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:21.610713 11982 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 12422
07:07:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:21.639369 11982 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 12567
07:07:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:21.667016 11982 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 11994
07:07:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:21.701164 11982 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 12064
07:07:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 07:07:21.732913 11982 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskLw1kUt/build/tsan/bin/kudu with pid 12139
07:07:22.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T07:07:22Z chronyd exiting
07:07:23.847 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=27987), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [246ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [263ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [599ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [604ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [627ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [628ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [630ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [643ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [643ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [645ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [655ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [660ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [661ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [662ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [680ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [682ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [682ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [700ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [702ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [704ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [780ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [780ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [782ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [821ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [822ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [823ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [855ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [859ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [861ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [862ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [863ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1015ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1026ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1795ms] refreshing cache from master, [1795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [1796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [1797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [1799ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1799ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [1801ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [1802ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [3215ms] refreshing cache from master, [3215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [3216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [3217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [3220ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [3221ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [3222ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [3223ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [4675ms] refreshing cache from master, [4675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [4679ms] too many traces: truncated at 100 traces, deferred=Deferred@25312984(state=PENDING, result=null, callback=(continuation of Deferred@2080100185 after retry RPC after error@691316415) -> (continuation of Deferred@296106798 after retry RPC after error@21144380) -> (continuation of Deferred@619062602 after retry RPC after error@678133303) -> (continuation of Deferred@146038623 after retry RPC after error@486229558) -> (continuation of Deferred@153287830 after retry RPC after error@2051776835) -> (continuation of Deferred@1353668594 after retry RPC after error@379805991) -> (continuation of Deferred@142342441 after retry RPC after error@798822620) -> (continuation of Deferred@94880282 after retry RPC after error@142619088) -> (continuation of Deferred@1600463704 after retry RPC after error@414831512) -> (continuation of Deferred@590706055 after retry RPC after error@716361344) -> (continuation of Deferred@1547388781 after retry RPC after error@180910689) -> (continuation of Deferred@47824882 after retry RPC after error@1042528231) -> (continuation of Deferred@950378337 after retry RPC after error@1903681894) -> (continuation of Deferred@1050601109 after retry RPC after error@488759854) -> (continuation of Deferred@166410364 after retry RPC after error@1197053174) -> (continuation of Deferred@247097542 after retry RPC after error@471391149) -> (continuation of Deferred@1281370250 after retry RPC after error@1201011668) -> (continuation of Deferred@439637944 after retry RPC after error@1076568800) -> (continuation of Deferred@222455319 after retry RPC after error@1709270832) -> (continuation of Deferred@1573942691 after retry RPC after error@1148547291) -> (continuation of Deferred@1520030876 after retry RPC after error@875556888) -> (continuation of Deferred@1779234387 after retry RPC after error@84979809) -> (continuation of Deferred@288995691 after retry RPC after error@1516053604) -> (continuation of Deferred@1466887954 after retry RPC after error@1467862566) -> (continuation of Deferred@190039875 after retry RPC after error@941538377) -> (continuation of Deferred@405328423 after retry RPC after error@555557882) -> (continuation of Deferred@887198021 after retry RPC after error@2043711332), errback=(continuation of Deferred@2080100185 after retry RPC after error@691316415) -> (continuation of Deferred@296106798 after retry RPC after error@21144380) -> (continuation of Deferred@619062602 after retry RPC after error@678133303) -> (continuation of Deferred@146038623 after retry RPC after error@486229558) -> (continuation of Deferred@153287830 after retry RPC after error@2051776835) -> (continuation of Deferred@1353668594 after retry RPC after error@379805991) -> (continuation of Deferred@142342441 after retry RPC after error@798822620) -> (continuation of Deferred@94880282 after retry RPC after error@142619088) -> (continuation of Deferred@1600463704 after retry RPC after error@414831512) -> (continuation of Deferred@590706055 after retry RPC after error@716361344) -> (continuation of Deferred@1547388781 after retry RPC after error@180910689) -> (continuation of Deferred@47824882 after retry RPC after error@1042528231) -> (continuation of Deferred@950378337 after retry RPC after error@1903681894) -> (continuation of Deferred@1050601109 after retry RPC after error@488759854) -> (continuation of Deferred@166410364 after retry RPC after error@1197053174) -> (continuation of Deferred@247097542 after retry RPC after error@471391149) -> (continuation of Deferred@1281370250 after retry RPC after error@1201011668) -> (continuation of Deferred@439637944 after retry RPC after error@1076568800) -> (continuation of Deferred@222455319 after retry RPC after error@1709270832) -> (continuation of Deferred@1573942691 after retry RPC after error@1148547291) -> (continuation of Deferred@1520030876 after retry RPC after error@875556888) -> (continuation of Deferred@1779234387 after retry RPC after error@84979809) -> (continuation of Deferred@288995691 after retry RPC after error@1516053604) -> (continuation of Deferred@1466887954 after retry RPC after error@1467862566) -> (continuation of Deferred@190039875 after retry RPC after error@941538377) -> (continuation of Deferred@405328423 after retry RPC after error@555557882) -> (continuation of Deferred@887198021 after retry RPC after error@2043711332)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.12.105.190:38807
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=27987), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [246ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [263ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [599ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [604ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [627ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [627ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [628ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [630ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [643ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [643ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [644ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [645ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [655ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [660ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [661ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [662ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [680ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [682ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [682ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [700ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [702ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [704ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [780ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [780ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [782ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [783ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [821ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [822ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [823ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [824ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [855ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [859ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [861ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [862ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [863ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1015ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1026ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1795ms] refreshing cache from master, [1795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [1796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [1797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [1799ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [1799ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [1801ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [1802ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [3215ms] refreshing cache from master, [3215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [3216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [3217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [3220ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.189:44215: Network error: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215, [3221ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.188:40859: OK, [3222ms] Sub RPC ConnectToMaster: received response from server master-127.12.105.190:38807: Network error: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [3223ms] delaying RPC due to: Service unavailable: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807, [4675ms] refreshing cache from master, [4675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.189:44215, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.188:40859, [4677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.105.190:38807, [4679ms] too many traces: truncated at 100 traces, deferred=Deferred@25312984(state=PENDING, result=null, callback=(continuation of Deferred@2080100185 after retry RPC after error@691316415) -> (continuation of Deferred@296106798 after retry RPC after error@21144380) -> (continuation of Deferred@619062602 after retry RPC after error@678133303) -> (continuation of Deferred@146038623 after retry RPC after error@486229558) -> (continuation of Deferred@153287830 after retry RPC after error@2051776835) -> (continuation of Deferred@1353668594 after retry RPC after error@379805991) -> (continuation of Deferred@142342441 after retry RPC after error@798822620) -> (continuation of Deferred@94880282 after retry RPC after error@142619088) -> (continuation of Deferred@1600463704 after retry RPC after error@414831512) -> (continuation of Deferred@590706055 after retry RPC after error@716361344) -> (continuation of Deferred@1547388781 after retry RPC after error@180910689) -> (continuation of Deferred@47824882 after retry RPC after error@1042528231) -> (continuation of Deferred@950378337 after retry RPC after error@1903681894) -> (continuation of Deferred@1050601109 after retry RPC after error@488759854) -> (continuation of Deferred@166410364 after retry RPC after error@1197053174) -> (continuation of Deferred@247097542 after retry RPC after error@471391149) -> (continuation of Deferred@1281370250 after retry RPC after error@1201011668) -> (continuation of Deferred@439637944 after retry RPC after error@1076568800) -> (continuation of Deferred@222455319 after retry RPC after error@1709270832) -> (continuation of Deferred@1573942691 after retry RPC after error@1148547291) -> (continuation of Deferred@1520030876 after retry RPC after error@875556888) -> (continuation of Deferred@1779234387 after retry RPC after error@84979809) -> (continuation of Deferred@288995691 after retry RPC after error@1516053604) -> (continuation of Deferred@1466887954 after retry RPC after error@1467862566) -> (continuation of Deferred@190039875 after retry RPC after error@941538377) -> (continuation of Deferred@405328423 after retry RPC after error@555557882) -> (continuation of Deferred@887198021 after retry RPC after error@2043711332), errback=(continuation of Deferred@2080100185 after retry RPC after error@691316415) -> (continuation of Deferred@296106798 after retry RPC after error@21144380) -> (continuation of Deferred@619062602 after retry RPC after error@678133303) -> (continuation of Deferred@146038623 after retry RPC after error@486229558) -> (continuation of Deferred@153287830 after retry RPC after error@2051776835) -> (continuation of Deferred@1353668594 after retry RPC after error@379805991) -> (continuation of Deferred@142342441 after retry RPC after error@798822620) -> (continuation of Deferred@94880282 after retry RPC after error@142619088) -> (continuation of Deferred@1600463704 after retry RPC after error@414831512) -> (continuation of Deferred@590706055 after retry RPC after error@716361344) -> (continuation of Deferred@1547388781 after retry RPC after error@180910689) -> (continuation of Deferred@47824882 after retry RPC after error@1042528231) -> (continuation of Deferred@950378337 after retry RPC after error@1903681894) -> (continuation of Deferred@1050601109 after retry RPC after error@488759854) -> (continuation of Deferred@166410364 after retry RPC after error@1197053174) -> (continuation of Deferred@247097542 after retry RPC after error@471391149) -> (continuation of Deferred@1281370250 after retry RPC after error@1201011668) -> (continuation of Deferred@439637944 after retry RPC after error@1076568800) -> (continuation of Deferred@222455319 after retry RPC after error@1709270832) -> (continuation of Deferred@1573942691 after retry RPC after error@1148547291) -> (continuation of Deferred@1520030876 after retry RPC after error@875556888) -> (continuation of Deferred@1779234387 after retry RPC after error@84979809) -> (continuation of Deferred@288995691 after retry RPC after error@1516053604) -> (continuation of Deferred@1466887954 after retry RPC after error@1467862566) -> (continuation of Deferred@190039875 after retry RPC after error@941538377) -> (continuation of Deferred@405328423 after retry RPC after error@555557882) -> (continuation of Deferred@887198021 after retry RPC after error@2043711332)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.12.105.189:44215,127.12.105.188:40859,127.12.105.190:38807) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.189:44215(127.12.105.189:44215): Connection refused: /127.12.105.189:44215,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.12.105.190:38807(127.12.105.190:38807): Connection refused: /127.12.105.190:38807
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.12.105.190:38807
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)