13:37:43.736 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
13:37:43.736 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbfnQDJ/test-tmp
13:37:43.736 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:37:43.736 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbfnQDJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:37:43.737 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
13:37:44.041 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:37:44.041 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:37:44.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.044274 16142 mini_oidc.cc:152] Starting JWKS server
13:37:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.047504 16142 webserver.cc:460] Webserver started at http://127.0.0.1:39781/ using document root <none> and password file <none>
13:37:44.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.047927 16142 mini_oidc.cc:189] Starting OIDC Discovery server
13:37:44.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.049422 16142 webserver.cc:460] Webserver started at http://127.0.0.1:37021/ using document root <none> and password file <none>
13:37:44.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:37:44Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:37:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:37:44Z Disabled control of system clock
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.108233 16142 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/wal
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/data
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/logs
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/data/info.pb
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.190:33393
13:37:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.190
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39781/jwks.json with env {}
13:37:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.386305 16154 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:37:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.386816 16154 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:37:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.387275 16154 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:37:44.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.415380 16154 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:37:44.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.415639 16154 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:37:44.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.415833 16154 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:37:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.416086 16154 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:37:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.416287 16154 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:37:44.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.416426 16154 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39781/jwks.json
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.447742 16154 master_runner.cc:386] Master server non-default flags:
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/data
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/wal
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.190:33393
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39781/jwks.json
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/data/info.pb
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.190
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/logs
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:37:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:37:44.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.448844 16154 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:37:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.450274 16154 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:37:44.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.464668 16161 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:44.466199 16154 server_base.cc:1034] running on GCE node
13:37:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.465672 16163 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:44.464998 16160 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:45.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.517516 16154 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:37:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.520490 16154 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:37:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.521886 16154 hybrid_clock.cc:648] HybridClock initialized: now 1714138665521837 us; error 67 us; skew 500 ppm
13:37:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.522684 16154 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:37:45.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.529253 16154 webserver.cc:460] Webserver started at http://127.15.195.190:40199/ using document root <none> and password file <none>
13:37:45.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.530097 16154 fs_manager.cc:359] Metadata directory not provided
13:37:45.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.530277 16154 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:37:45.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.530645 16154 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:37:45.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.535315 16154 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/data/instance:
13:37:45.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a35fd2f6709a461daabd6ab9152fa4b9"
13:37:45.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:45 on dist-test-slave-6qgw"
13:37:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.536332 16154 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/wal/instance:
13:37:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a35fd2f6709a461daabd6ab9152fa4b9"
13:37:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:45 on dist-test-slave-6qgw"
13:37:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.543056 16154 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.003s
13:37:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.548272 16170 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:37:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.549197 16154 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
13:37:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.549506 16154 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/wal
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a35fd2f6709a461daabd6ab9152fa4b9"
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:45 on dist-test-slave-6qgw"
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.549793 16154 fs_report.cc:383] FS layout report
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/wal
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/wal
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/data/data
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:37:45.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:37:45.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.604259 16154 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:37:45.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.605600 16154 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:37:45.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.606001 16154 kserver.cc:163] Server-wide thread pool size limit: 3276
13:37:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.672196 16154 rpc_server.cc:307] RPC server started. Bound to: 127.15.195.190:33393
13:37:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.672294 16221 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.195.190:33393 every 8 connection(s)
13:37:45.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.674794 16154 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/data/info.pb
13:37:45.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.678884 16142 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 16154
13:37:45.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.679294 16142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-0/wal/instance
13:37:45.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.679697 16222 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.684098 16142 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/wal
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/data
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/logs
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/data/info.pb
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.189:36313
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.189
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:45.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39781/jwks.json with env {}
13:37:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.696965 16222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } has no permanent_uuid. Determining permanent_uuid...
13:37:45.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.724253 16222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } has no permanent_uuid. Determining permanent_uuid...
13:37:45.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:45.726673 16172 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.195.189:36313: connect: Connection refused (error 111)
13:37:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:45.730595 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.189:36313: Network error: Client connection negotiation failed: client connection to 127.15.195.189:36313: connect: Connection refused (error 111)
13:37:45.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.779994 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } attempt: 1
13:37:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:45.784029 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.189:36313: Network error: Client connection negotiation failed: client connection to 127.15.195.189:36313: connect: Connection refused (error 111)
13:37:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.852365 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } attempt: 2
13:37:45.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:45.855767 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.189:36313: Network error: Client connection negotiation failed: client connection to 127.15.195.189:36313: connect: Connection refused (error 111)
13:37:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:45.947237 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } attempt: 3
13:37:45.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:45.951582 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.189:36313: Network error: Client connection negotiation failed: client connection to 127.15.195.189:36313: connect: Connection refused (error 111)
13:37:45.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:45.978569 16224 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:37:45.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:45.979218 16224 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:37:45.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:45.979739 16224 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:37:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.006924 16224 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:37:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.007234 16224 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:37:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.007503 16224 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:37:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.007761 16224 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:37:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.008031 16224 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:37:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.008232 16224 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39781/jwks.json
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:46.038781 16224 master_runner.cc:386] Master server non-default flags:
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/data
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/wal
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.189:36313
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39781/jwks.json
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/data/info.pb
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.189
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/logs
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:37:46.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:37:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:37:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:37:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:37:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:37:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:37:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:37:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:46.040176 16224 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:37:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:46.041833 16224 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:37:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.055840 16233 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:46.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:46.094905 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } attempt: 4
13:37:46.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.098914 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.189:36313: Network error: Client connection negotiation failed: client connection to 127.15.195.189:36313: connect: Connection refused (error 111)
13:37:46.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:46.398502 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } attempt: 5
13:37:46.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.402329 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.189:36313: Network error: Client connection negotiation failed: client connection to 127.15.195.189:36313: connect: Connection refused (error 111)
13:37:46.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:46.949880 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } attempt: 6
13:37:46.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.956372 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.189:36313: Network error: Client connection negotiation failed: client connection to 127.15.195.189:36313: connect: Connection refused (error 111)
13:37:47.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.056231 16236 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:47.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:46.056926 16234 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:47.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:46.057221 16224 server_base.cc:1034] running on GCE node
13:37:47.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.114246 16224 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:37:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.117215 16224 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:37:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.118649 16224 hybrid_clock.cc:648] HybridClock initialized: now 1714138667118625 us; error 62 us; skew 500 ppm
13:37:47.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.119387 16224 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:37:47.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.125193 16224 webserver.cc:460] Webserver started at http://127.15.195.189:34357/ using document root <none> and password file <none>
13:37:47.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.126152 16224 fs_manager.cc:359] Metadata directory not provided
13:37:47.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.126343 16224 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:37:47.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.126703 16224 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:37:47.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.131201 16224 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/data/instance:
13:37:47.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "faf57c70c0df43708f2a898c04e37850"
13:37:47.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:47 on dist-test-slave-6qgw"
13:37:47.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.132125 16224 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/wal/instance:
13:37:47.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "faf57c70c0df43708f2a898c04e37850"
13:37:47.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:47 on dist-test-slave-6qgw"
13:37:47.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.138595 16224 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
13:37:47.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.143306 16245 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:37:47.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.144202 16224 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
13:37:47.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.144466 16224 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/wal
13:37:47.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "faf57c70c0df43708f2a898c04e37850"
13:37:47.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:47 on dist-test-slave-6qgw"
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.144734 16224 fs_report.cc:383] FS layout report
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/wal
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/wal
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/data/data
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:37:47.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:37:47.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.192387 16224 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:37:47.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.193637 16224 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:37:47.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.194044 16224 kserver.cc:163] Server-wide thread pool size limit: 3276
13:37:47.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.259310 16224 rpc_server.cc:307] RPC server started. Bound to: 127.15.195.189:36313
13:37:47.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.259411 16296 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.195.189:36313 every 8 connection(s)
13:37:47.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.261824 16224 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/data/info.pb
13:37:47.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.266621 16297 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:37:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.271445 16142 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 16224
13:37:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.271822 16142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-1/wal/instance
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.273733 16142 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/wal
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/data
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/logs
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/data/info.pb
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.188:34971
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.188
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:47.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:47.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39781/jwks.json with env {}
13:37:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.277380 16297 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } has no permanent_uuid. Determining permanent_uuid...
13:37:47.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.297376 16297 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } has no permanent_uuid. Determining permanent_uuid...
13:37:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.309764 16297 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } has no permanent_uuid. Determining permanent_uuid...
13:37:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.311815 16246 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:47.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.315295 16297 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:47.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.364841 16297 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 1
13:37:47.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.368755 16297 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.437208 16297 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 2
13:37:47.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.440748 16297 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:47.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.532176 16297 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 3
13:37:47.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.537060 16297 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:47.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.566964 16299 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:37:47.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.567543 16299 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:37:47.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.568078 16299 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:37:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.596530 16299 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:37:47.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.596848 16299 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:37:47.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.597147 16299 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:37:47.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.597436 16299 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:37:47.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.597723 16299 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:37:47.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.597923 16299 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39781/jwks.json
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.628978 16299 master_runner.cc:386] Master server non-default flags:
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/data
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/wal
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.188:34971
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39781/jwks.json
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/data/info.pb
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.188
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:37:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/logs
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:37:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.630440 16299 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:37:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.632066 16299 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:37:47.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.648187 16309 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.680444 16297 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 4
13:37:47.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.686138 16297 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:47.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.985571 16297 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 5
13:37:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.990804 16297 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:48.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.016841 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } attempt: 7
13:37:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.037837 16222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } has no permanent_uuid. Determining permanent_uuid...
13:37:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:48.045123 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:48.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.103508 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 1
13:37:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:48.108371 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:48.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.189723 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 2
13:37:48.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:48.194583 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.279973 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 3
13:37:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:48.285342 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:48.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.425758 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 4
13:37:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:48.430866 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:48.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.538178 16297 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 6
13:37:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:48.543818 16297 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:48.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.714241 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 5
13:37:48.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:48.718154 16222 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.195.188:34971: Network error: Client connection negotiation failed: client connection to 127.15.195.188:34971: connect: Connection refused (error 111)
13:37:48.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.648523 16312 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:48.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:47.649519 16299 server_base.cc:1034] running on GCE node
13:37:48.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:47.649178 16310 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:48.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.734537 16299 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:37:48.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.737521 16299 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:37:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.738917 16299 hybrid_clock.cc:648] HybridClock initialized: now 1714138668738872 us; error 50 us; skew 500 ppm
13:37:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.739652 16299 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:37:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.745405 16299 webserver.cc:460] Webserver started at http://127.15.195.188:37419/ using document root <none> and password file <none>
13:37:48.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.746332 16299 fs_manager.cc:359] Metadata directory not provided
13:37:48.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.746520 16299 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:37:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.746874 16299 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:37:48.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.751612 16299 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/data/instance:
13:37:48.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984"
13:37:48.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:48 on dist-test-slave-6qgw"
13:37:48.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.752614 16299 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/wal/instance:
13:37:48.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984"
13:37:48.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:48 on dist-test-slave-6qgw"
13:37:48.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.759542 16299 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.007s
13:37:48.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.764568 16324 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:37:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.765599 16299 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.765904 16299 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/wal
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984"
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:48 on dist-test-slave-6qgw"
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.766199 16299 fs_report.cc:383] FS layout report
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/wal
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/wal
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/data/data
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:37:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:37:48.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.815876 16299 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:37:48.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.817204 16299 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:37:48.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.817572 16299 kserver.cc:163] Server-wide thread pool size limit: 3276
13:37:48.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.883304 16299 rpc_server.cc:307] RPC server started. Bound to: 127.15.195.188:34971
13:37:48.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.883370 16375 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.195.188:34971 every 8 connection(s)
13:37:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.885876 16299 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/data/info.pb
13:37:48.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.888268 16142 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 16299
13:37:48.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.888761 16142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/master-2/wal/instance
13:37:48.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.890935 16376 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:37:48.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.909351 16376 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } has no permanent_uuid. Determining permanent_uuid...
13:37:48.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.932606 16376 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } has no permanent_uuid. Determining permanent_uuid...
13:37:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.941028 16376 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } has no permanent_uuid. Determining permanent_uuid...
13:37:48.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.955708 16376 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: Bootstrap starting.
13:37:48.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.961773 16376 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: Neither blocks nor log segments found. Creating new log.
13:37:48.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.963308 16376 log.cc:824] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: Log is configured to *not* fsync() on all Append() calls
13:37:48.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.967437 16376 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: No bootstrap required, opened a new log
13:37:48.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.988477 16376 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:48.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.988996 16376 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:37:48.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.989163 16376 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dc3f57a63d14ba9b3a3589aa8cfe984, State: Initialized, Role: FOLLOWER
13:37:48.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.989765 16376 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:48.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.991842 16385 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:48.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.992481 16385 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [sys.catalog]: This master's current role is: FOLLOWER
13:37:48.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:48.993149 16376 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [sys.catalog]: configured and running, proceeding with master startup.
13:37:49.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:49.015451 16396 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:37:49.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:49.015772 16396 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:37:49.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.010850 16299 master_runner.cc:186] Error getting master registration for 127.15.195.190:33393: OK, instance_id { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" instance_seqno: 1714138665654077 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:37:49.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.033483 16299 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:37:49.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:49.072000 16330 tablet.cc:2369] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:37:49.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.270535 16222 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 6
13:37:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.290127 16222 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9: Bootstrap starting.
13:37:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.295320 16222 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9: Neither blocks nor log segments found. Creating new log.
13:37:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.297117 16222 log.cc:824] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9: Log is configured to *not* fsync() on all Append() calls
13:37:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.301518 16222 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9: No bootstrap required, opened a new log
13:37:49.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.324090 16222 raft_consensus.cc:348] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:49.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.324975 16222 raft_consensus.cc:374] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:37:49.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.325295 16222 raft_consensus.cc:729] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a35fd2f6709a461daabd6ab9152fa4b9, State: Initialized, Role: FOLLOWER
13:37:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.326165 16222 consensus_queue.cc:260] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:49.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.329330 16400 sys_catalog.cc:455] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:49.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.330137 16400 sys_catalog.cc:458] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: This master's current role is: FOLLOWER
13:37:49.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.330740 16222 sys_catalog.cc:564] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: configured and running, proceeding with master startup.
13:37:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.341498 16400 raft_consensus.cc:482] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:37:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.341989 16400 raft_consensus.cc:504] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.344686 16400 leader_election.cc:288] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers faf57c70c0df43708f2a898c04e37850 (127.15.195.189:36313), 0dc3f57a63d14ba9b3a3589aa8cfe984 (127.15.195.188:34971)
13:37:49.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.346843 16351 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" is_pre_election: true
13:37:49.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.347735 16351 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a35fd2f6709a461daabd6ab9152fa4b9 in term 0.
13:37:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.349390 16171 leader_election.cc:302] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0dc3f57a63d14ba9b3a3589aa8cfe984, a35fd2f6709a461daabd6ab9152fa4b9; no voters:
13:37:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.350216 16400 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 0 FOLLOWER]: Leader pre-election won for term 1
13:37:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.350543 16400 raft_consensus.cc:482] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:37:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.350831 16400 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 0 FOLLOWER]: Advancing to term 1
13:37:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.357160 16400 raft_consensus.cc:504] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.359705 16351 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984"
13:37:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.360178 16351 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [term 0 FOLLOWER]: Advancing to term 1
13:37:49.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:49.363852 16176 tablet.cc:2369] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:37:49.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.366464 16400 leader_election.cc:288] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [CANDIDATE]: Term 1 election: Requested vote from peers faf57c70c0df43708f2a898c04e37850 (127.15.195.189:36313), 0dc3f57a63d14ba9b3a3589aa8cfe984 (127.15.195.188:34971)
13:37:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.367076 16351 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a35fd2f6709a461daabd6ab9152fa4b9 in term 1.
13:37:49.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.368259 16171 leader_election.cc:302] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0dc3f57a63d14ba9b3a3589aa8cfe984, a35fd2f6709a461daabd6ab9152fa4b9; no voters:
13:37:49.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.369074 16400 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 1 FOLLOWER]: Leader election won for term 1
13:37:49.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.371591 16400 raft_consensus.cc:686] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [term 1 LEADER]: Becoming Leader. State: Replica: a35fd2f6709a461daabd6ab9152fa4b9, State: Running, Role: LEADER
13:37:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.372644 16400 consensus_queue.cc:237] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:49.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.391346 16412 sys_catalog.cc:455] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a35fd2f6709a461daabd6ab9152fa4b9. Latest consensus state: current_term: 1 leader_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:49.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.392236 16412 sys_catalog.cc:458] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: This master's current role is: LEADER
13:37:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:49.397713 16411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:37:49.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:49.398026 16411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:37:49.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.402617 16415 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:37:49.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.409757 16415 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:37:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.397863 16154 master_runner.cc:186] Error getting master registration for 127.15.195.189:36313: OK, instance_id { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" instance_seqno: 1714138667240841 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:37:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.431751 16154 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:37:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.442772 16351 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [term 1 FOLLOWER]: Refusing update from remote peer a35fd2f6709a461daabd6ab9152fa4b9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:37:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.444725 16412 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:37:49.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.476305 16385 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a35fd2f6709a461daabd6ab9152fa4b9. Latest consensus state: current_term: 1 leader_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:49.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.480443 16385 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [sys.catalog]: This master's current role is: FOLLOWER
13:37:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.484900 16416 mvcc.cc:205] Tried to move back new op lower bound from 7021111990019252224 to 7021111989762629632. Current Snapshot: MvccSnapshot[applied={T|T < 7021111990019252224}]
13:37:49.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.490268 16400 sys_catalog.cc:455] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:49.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.491191 16400 sys_catalog.cc:458] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: This master's current role is: LEADER
13:37:49.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.492998 16415 catalog_manager.cc:1347] Generated new cluster ID: 44ebc9ad872c45b48d12d3aade00fa43
13:37:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.493278 16415 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:37:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.493889 16385 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.494594 16385 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984 [sys.catalog]: This master's current role is: FOLLOWER
13:37:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.526398 16415 catalog_manager.cc:1370] Generated new certificate authority record
13:37:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.530859 16415 catalog_manager.cc:1504] Loading token signing keys...
13:37:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.560848 16415 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9: Generated new TSK 0
13:37:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.564744 16415 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.594650 16142 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/wal
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/data
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/logs
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/data/info.pb
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.129:0
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.195.129
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.129
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:37:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.604142 16297 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } attempt: 7
13:37:49.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.646479 16297 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850: Bootstrap starting.
13:37:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.660387 16297 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850: Neither blocks nor log segments found. Creating new log.
13:37:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.662703 16297 log.cc:824] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850: Log is configured to *not* fsync() on all Append() calls
13:37:49.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.679443 16297 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850: No bootstrap required, opened a new log
13:37:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.731829 16297 raft_consensus.cc:348] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.732703 16297 raft_consensus.cc:374] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:37:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.733004 16297 raft_consensus.cc:729] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: faf57c70c0df43708f2a898c04e37850, State: Initialized, Role: FOLLOWER
13:37:49.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.745922 16297 consensus_queue.cc:260] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } }
13:37:49.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.751108 16272 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "faf57c70c0df43708f2a898c04e37850" is_pre_election: true
13:37:49.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.752800 16271 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "faf57c70c0df43708f2a898c04e37850"
13:37:49.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.752213 16272 raft_consensus.cc:2454] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a35fd2f6709a461daabd6ab9152fa4b9 in term 0.
13:37:49.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.750629 16297 sys_catalog.cc:564] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [sys.catalog]: configured and running, proceeding with master startup.
13:37:49.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.757454 16270 raft_consensus.cc:3037] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [term 0 FOLLOWER]: Advancing to term 1
13:37:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.764807 16270 raft_consensus.cc:1261] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [term 1 FOLLOWER]: Refusing update from remote peer a35fd2f6709a461daabd6ab9152fa4b9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:37:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.768034 16412 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:37:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.767925 16424 sys_catalog.cc:455] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.768642 16424 sys_catalog.cc:458] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [sys.catalog]: This master's current role is: FOLLOWER
13:37:49.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.870965 16430 sys_catalog.cc:455] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:49.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.872082 16430 sys_catalog.cc:458] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [sys.catalog]: This master's current role is: FOLLOWER
13:37:49.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.872910 16424 sys_catalog.cc:455] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a35fd2f6709a461daabd6ab9152fa4b9. Latest consensus state: current_term: 1 leader_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:49.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.874258 16424 sys_catalog.cc:458] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850 [sys.catalog]: This master's current role is: FOLLOWER
13:37:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.929822 16440 catalog_manager.cc:1259] Loaded cluster ID: 44ebc9ad872c45b48d12d3aade00fa43
13:37:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.930083 16440 catalog_manager.cc:1552] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850: loading cluster ID for follower catalog manager: success
13:37:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.952263 16440 catalog_manager.cc:1574] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850: acquiring CA information for follower catalog manager: success
13:37:49.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:49.968889 16440 catalog_manager.cc:1602] T 00000000000000000000000000000000 P faf57c70c0df43708f2a898c04e37850: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:37:50.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.021353 16396 catalog_manager.cc:1259] Loaded cluster ID: 44ebc9ad872c45b48d12d3aade00fa43
13:37:50.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.021843 16396 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: loading cluster ID for follower catalog manager: success
13:37:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.030066 16396 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: acquiring CA information for follower catalog manager: success
13:37:50.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.050060 16396 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0dc3f57a63d14ba9b3a3589aa8cfe984: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:37:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:50.136750 16421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:37:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:50.137599 16421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:37:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:50.138475 16421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:37:50.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:50.190097 16421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:37:50.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:50.191581 16421 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.195.129
13:37:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.248054 16421 tablet_server_runner.cc:78] Tablet server non-default flags:
13:37:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/data
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/wal
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.129:0
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/data/info.pb
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.129
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.195.129
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/logs
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:37:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:37:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.251196 16421 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:37:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.253331 16421 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:37:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:50.293615 16447 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.386263 16412 sys_catalog.cc:455] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a35fd2f6709a461daabd6ab9152fa4b9" member_type: VOTER last_known_addr { host: "127.15.195.190" port: 33393 } } peers { permanent_uuid: "faf57c70c0df43708f2a898c04e37850" member_type: VOTER last_known_addr { host: "127.15.195.189" port: 36313 } } peers { permanent_uuid: "0dc3f57a63d14ba9b3a3589aa8cfe984" member_type: VOTER last_known_addr { host: "127.15.195.188" port: 34971 } } }
13:37:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:50.387207 16412 sys_catalog.cc:458] T 00000000000000000000000000000000 P a35fd2f6709a461daabd6ab9152fa4b9 [sys.catalog]: This master's current role is: LEADER
13:37:52.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:51.674386 16446 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 16421
13:37:52.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:52.639008 16421 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.343s user 0.344s sys 0.825s
13:37:52.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:52.640657 16421 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.344s user 0.344s sys 0.825s
13:37:52.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:52.639746 16449 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2343 milliseconds
13:37:52.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:50.302006 16448 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:52.642107 16451 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.642077 16421 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:37:52.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.646312 16421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:37:52.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.649021 16421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:37:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.650563 16421 hybrid_clock.cc:648] HybridClock initialized: now 1714138672650524 us; error 47 us; skew 500 ppm
13:37:52.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.651356 16421 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:37:52.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.658332 16421 webserver.cc:460] Webserver started at http://127.15.195.129:44843/ using document root <none> and password file <none>
13:37:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.659243 16421 fs_manager.cc:359] Metadata directory not provided
13:37:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.659444 16421 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:37:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.659925 16421 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:37:52.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.665343 16421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/data/instance:
13:37:52.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d747e165fb794373979ca39559463fb4"
13:37:52.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:52 on dist-test-slave-6qgw"
13:37:52.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.666520 16421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/wal/instance:
13:37:52.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d747e165fb794373979ca39559463fb4"
13:37:52.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:52 on dist-test-slave-6qgw"
13:37:52.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.679941 16421 fs_manager.cc:691] Time spent creating directory manager: real 0.013s user 0.009s sys 0.003s
13:37:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.693226 16460 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:37:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.694674 16421 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.007s sys 0.005s
13:37:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.694999 16421 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/wal
13:37:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d747e165fb794373979ca39559463fb4"
13:37:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:52 on dist-test-slave-6qgw"
13:37:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.695427 16421 fs_report.cc:383] FS layout report
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/wal
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/wal
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/data/data
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:37:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:37:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.759757 16421 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:37:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.761283 16421 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:37:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.761749 16421 kserver.cc:163] Server-wide thread pool size limit: 3276
13:37:52.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.764679 16421 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:37:52.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.769282 16421 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:37:52.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.769491 16421 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:37:52.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.769755 16421 ts_tablet_manager.cc:590] Registered 0 tablets
13:37:52.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.769914 16421 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:37:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.957322 16421 rpc_server.cc:307] RPC server started. Bound to: 127.15.195.129:46551
13:37:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.957427 16573 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.195.129:46551 every 8 connection(s)
13:37:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.959764 16421 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/data/info.pb
13:37:52.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.969887 16142 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 16421
13:37:52.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.970314 16142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-0/wal/instance
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:52.978735 16142 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/wal
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/data
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/logs
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/data/info.pb
13:37:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.130:0
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.195.130
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.130
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:37:53.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.007344 16576 heartbeater.cc:347] Connected to a master server at 127.15.195.190:33393
13:37:53.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.007730 16576 heartbeater.cc:464] Registering TS with master...
13:37:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.008888 16576 heartbeater.cc:511] Master 127.15.195.190:33393 requested a full tablet report, sending...
13:37:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.011700 16187 ts_manager.cc:194] Registered new tserver with Master: d747e165fb794373979ca39559463fb4 (127.15.195.129:46551)
13:37:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.014406 16187 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.195.129:34379
13:37:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.015210 16575 heartbeater.cc:347] Connected to a master server at 127.15.195.188:34971
13:37:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.015513 16575 heartbeater.cc:464] Registering TS with master...
13:37:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.016340 16575 heartbeater.cc:511] Master 127.15.195.188:34971 requested a full tablet report, sending...
13:37:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.019073 16341 ts_manager.cc:194] Registered new tserver with Master: d747e165fb794373979ca39559463fb4 (127.15.195.129:46551)
13:37:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.019682 16574 heartbeater.cc:347] Connected to a master server at 127.15.195.189:36313
13:37:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.019959 16574 heartbeater.cc:464] Registering TS with master...
13:37:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.020583 16574 heartbeater.cc:511] Master 127.15.195.189:36313 requested a full tablet report, sending...
13:37:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.023582 16262 ts_manager.cc:194] Registered new tserver with Master: d747e165fb794373979ca39559463fb4 (127.15.195.129:46551)
13:37:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:53.305497 16582 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:37:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:53.305963 16582 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:37:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:53.306389 16582 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:37:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:53.340432 16582 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:37:53.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:53.341243 16582 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.195.130
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.371663 16582 tablet_server_runner.cc:78] Tablet server non-default flags:
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/data
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/wal
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.130:0
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/data/info.pb
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.130
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:37:53.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.195.130
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/logs
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:37:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.372714 16582 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:37:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.374264 16582 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:37:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:53.391577 16590 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:54.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.019521 16576 heartbeater.cc:503] Master 127.15.195.190:33393 was elected leader, sending a full tablet report...
13:37:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:53.391993 16593 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:53.392545 16591 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:53.392978 16582 server_base.cc:1034] running on GCE node
13:37:54.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.468477 16582 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:37:54.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.471295 16582 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:37:54.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.472819 16582 hybrid_clock.cc:648] HybridClock initialized: now 1714138674472777 us; error 55 us; skew 500 ppm
13:37:54.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.473583 16582 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:37:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.484304 16582 webserver.cc:460] Webserver started at http://127.15.195.130:35653/ using document root <none> and password file <none>
13:37:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.485109 16582 fs_manager.cc:359] Metadata directory not provided
13:37:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.485285 16582 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:37:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.485688 16582 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:37:54.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.490604 16582 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/data/instance:
13:37:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e48bdbb0275a47b583c145ea0e0c34cc"
13:37:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:54 on dist-test-slave-6qgw"
13:37:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.491534 16582 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/wal/instance:
13:37:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e48bdbb0275a47b583c145ea0e0c34cc"
13:37:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:54 on dist-test-slave-6qgw"
13:37:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.498507 16582 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
13:37:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.503571 16602 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:37:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.504576 16582 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.004s
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.504817 16582 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/wal
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e48bdbb0275a47b583c145ea0e0c34cc"
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:54 on dist-test-slave-6qgw"
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.505096 16582 fs_report.cc:383] FS layout report
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/wal
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/wal
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/data/data
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:37:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:37:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:37:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:37:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:37:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:37:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.555668 16582 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:37:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.557008 16582 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:37:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.557404 16582 kserver.cc:163] Server-wide thread pool size limit: 3276
13:37:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.559760 16582 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:37:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.563994 16582 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:37:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.564198 16582 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:37:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.564381 16582 ts_tablet_manager.cc:590] Registered 0 tablets
13:37:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.564522 16582 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:37:54.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.703224 16582 rpc_server.cc:307] RPC server started. Bound to: 127.15.195.130:44033
13:37:54.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.703395 16714 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.195.130:44033 every 8 connection(s)
13:37:54.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.705755 16582 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/data/info.pb
13:37:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.716424 16142 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 16582
13:37:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.717190 16142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-1/wal/instance
13:37:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.747417 16142 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/wal
13:37:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/data
13:37:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:37:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/logs
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/data/info.pb
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.131:0
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.195.131
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.131
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:37:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.758338 16715 heartbeater.cc:347] Connected to a master server at 127.15.195.189:36313
13:37:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.760928 16715 heartbeater.cc:464] Registering TS with master...
13:37:54.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.762163 16715 heartbeater.cc:511] Master 127.15.195.189:36313 requested a full tablet report, sending...
13:37:54.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.765105 16262 ts_manager.cc:194] Registered new tserver with Master: e48bdbb0275a47b583c145ea0e0c34cc (127.15.195.130:44033)
13:37:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.769001 16716 heartbeater.cc:347] Connected to a master server at 127.15.195.188:34971
13:37:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.769393 16716 heartbeater.cc:464] Registering TS with master...
13:37:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.770417 16716 heartbeater.cc:511] Master 127.15.195.188:34971 requested a full tablet report, sending...
13:37:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.770584 16717 heartbeater.cc:347] Connected to a master server at 127.15.195.190:33393
13:37:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.770910 16717 heartbeater.cc:464] Registering TS with master...
13:37:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.771710 16717 heartbeater.cc:511] Master 127.15.195.190:33393 requested a full tablet report, sending...
13:37:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.773239 16341 ts_manager.cc:194] Registered new tserver with Master: e48bdbb0275a47b583c145ea0e0c34cc (127.15.195.130:44033)
13:37:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.773432 16187 ts_manager.cc:194] Registered new tserver with Master: e48bdbb0275a47b583c145ea0e0c34cc (127.15.195.130:44033)
13:37:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:54.774945 16187 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.195.130:48461
13:37:55.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:55.048198 16725 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:37:55.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:55.048708 16725 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:37:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:55.049234 16725 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:37:55.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:55.077153 16725 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:37:55.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:55.077939 16725 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.195.131
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:55.110103 16725 tablet_server_runner.cc:78] Tablet server non-default flags:
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.195.148:39869
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/data
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/wal
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.195.131:0
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/data/info.pb
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.195.131
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.195.190:33393,127.15.195.189:36313,127.15.195.188:34971
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:37:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.195.131
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/logs
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:37:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:55.111533 16725 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:37:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:55.113058 16725 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:37:55.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:55.129675 16734 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:55.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:55.778259 16717 heartbeater.cc:503] Master 127.15.195.190:33393 was elected leader, sending a full tablet report...
13:37:56.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:55.130007 16731 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:56.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:55.130761 16725 server_base.cc:1034] running on GCE node
13:37:56.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:55.130357 16732 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:56.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.215379 16725 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:37:56.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.217725 16725 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:37:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.219072 16725 hybrid_clock.cc:648] HybridClock initialized: now 1714138676219025 us; error 58 us; skew 500 ppm
13:37:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.219866 16725 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:37:56.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.226475 16725 webserver.cc:460] Webserver started at http://127.15.195.131:37217/ using document root <none> and password file <none>
13:37:56.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.227334 16725 fs_manager.cc:359] Metadata directory not provided
13:37:56.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.227522 16725 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:37:56.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.227918 16725 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:37:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.232705 16725 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/data/instance:
13:37:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84c336c709184745ad901733d2888e0e"
13:37:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:56 on dist-test-slave-6qgw"
13:37:56.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.233752 16725 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/wal/instance:
13:37:56.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84c336c709184745ad901733d2888e0e"
13:37:56.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:56 on dist-test-slave-6qgw"
13:37:56.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.240780 16725 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
13:37:56.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.246125 16742 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:37:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.247037 16725 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
13:37:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.247305 16725 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/wal
13:37:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84c336c709184745ad901733d2888e0e"
13:37:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:56 on dist-test-slave-6qgw"
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.247604 16725 fs_report.cc:383] FS layout report
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/wal
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/wal
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/data/data
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:37:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:37:56.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.303478 16725 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:37:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.304776 16725 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:37:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.305171 16725 kserver.cc:163] Server-wide thread pool size limit: 3276
13:37:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.307595 16725 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:37:56.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.311269 16725 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:37:56.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.311439 16725 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:37:56.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.311614 16725 ts_tablet_manager.cc:590] Registered 0 tablets
13:37:56.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.311735 16725 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:37:56.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.439661 16725 rpc_server.cc:307] RPC server started. Bound to: 127.15.195.131:32873
13:37:56.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.439752 16854 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.195.131:32873 every 8 connection(s)
13:37:56.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.442224 16725 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/data/info.pb
13:37:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.452975 16142 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 16725
13:37:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.453652 16142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster7315837261187716606/ts-2/wal/instance
13:37:56.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.482738 16859 heartbeater.cc:347] Connected to a master server at 127.15.195.190:33393
13:37:56.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.483096 16859 heartbeater.cc:464] Registering TS with master...
13:37:56.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.484061 16856 heartbeater.cc:347] Connected to a master server at 127.15.195.188:34971
13:37:56.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.484293 16859 heartbeater.cc:511] Master 127.15.195.190:33393 requested a full tablet report, sending...
13:37:56.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.484347 16856 heartbeater.cc:464] Registering TS with master...
13:37:56.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.485409 16856 heartbeater.cc:511] Master 127.15.195.188:34971 requested a full tablet report, sending...
13:37:56.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.486701 16855 heartbeater.cc:347] Connected to a master server at 127.15.195.189:36313
13:37:56.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.487020 16855 heartbeater.cc:464] Registering TS with master...
13:37:56.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.487552 16187 ts_manager.cc:194] Registered new tserver with Master: 84c336c709184745ad901733d2888e0e (127.15.195.131:32873)
13:37:56.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.487797 16855 heartbeater.cc:511] Master 127.15.195.189:36313 requested a full tablet report, sending...
13:37:56.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.487973 16341 ts_manager.cc:194] Registered new tserver with Master: 84c336c709184745ad901733d2888e0e (127.15.195.131:32873)
13:37:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.489063 16187 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.195.131:36111
13:37:56.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.490289 16262 ts_manager.cc:194] Registered new tserver with Master: 84c336c709184745ad901733d2888e0e (127.15.195.131:32873)
13:37:56.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.493898 16142 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
13:37:56.503 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:37:56.503 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
13:37:56.504 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.195.190"
port: 33393
}
bound_http_address {
host: "127.15.195.190"
port: 40199
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.195.189"
port: 36313
}
bound_http_address {
host: "127.15.195.189"
port: 34357
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.195.188"
port: 34971
}
bound_http_address {
host: "127.15.195.188"
port: 37419
}
}
}
13:37:56.504 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:37:56.505 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.15.195.129"
port: 46551
}
bound_http_address {
host: "127.15.195.129"
port: 44843
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.15.195.130"
port: 44033
}
bound_http_address {
host: "127.15.195.130"
port: 35653
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.15.195.131"
port: 32873
}
bound_http_address {
host: "127.15.195.131"
port: 37217
}
}
}
13:37:56.505 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
13:37:56.506 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster3381552631525434212/krb5kdc/krb5cc
13:37:56.506 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
13:37:56.507 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbfnQDJ/test-tmp
13:37:56.507 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:37:56.507 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbfnQDJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:37:56.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
13:37:56.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:37:56.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:37:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.840412 16866 mini_oidc.cc:152] Starting JWKS server
13:37:56.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.843734 16866 webserver.cc:460] Webserver started at http://127.0.0.1:38515/ using document root <none> and password file <none>
13:37:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.844188 16866 mini_oidc.cc:189] Starting OIDC Discovery server
13:37:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:56.845964 16866 webserver.cc:460] Webserver started at http://127.0.0.1:33553/ using document root <none> and password file <none>
13:37:56.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
13:37:56.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/principal' for realm 'KRBTEST.COM',
13:37:56.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
13:37:56.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:37:56 dist-test-slave-6qgw krb5kdc[16877](info): setting up network...
13:37:56.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
13:37:56.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:37:56 dist-test-slave-6qgw krb5kdc[16877](info): set up 2 sockets
13:37:56.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:37:56 dist-test-slave-6qgw krb5kdc[16877](info): commencing operation
13:37:56.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
13:37:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:57.492338 16859 heartbeater.cc:503] Master 127.15.195.190:33393 was elected leader, sending a full tablet report...
13:37:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:57.870947 16866 mini_kdc.cc:121] Time spent starting KDC: real 0.986s user 0.001s sys 0.015s
13:37:57.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
13:37:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:37:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
13:37:57.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
13:37:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:37:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
13:37:57.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
13:37:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:37:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
13:37:57.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:37:57.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/test-user.keytab.
13:37:57.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/test-user.keytab.
13:37:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:37:58 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138677, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:37:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:37:58.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:37:58Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:37:58.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:37:58Z Disabled control of system clock
13:37:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.120.190@KRBTEST.COM; defaulting to no policy
13:37:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:37:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.120.190@KRBTEST.COM" created.
13:37:58.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:37:58.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.120.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab.
13:37:58.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.120.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab.
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:58.066076 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/logs
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.190:39275
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.190
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.190
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:37:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:37:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:37:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.353402 16891 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:37:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.354028 16891 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:37:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.354677 16891 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:37:58.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.383332 16891 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:37:58.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.383569 16891 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:37:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.383963 16891 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:37:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.384239 16891 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:37:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.384485 16891 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:37:58.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.384730 16891 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:37:58.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.384915 16891 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:58.417021 16891 master_runner.cc:386] Master server non-default flags:
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.190
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.190:39275
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:37:58.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.190
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/logs
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:37:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:58.418555 16891 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:37:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:58.419589 16891 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:37:58.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.435719 16897 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:59.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.436247 16898 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:59.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:58.436666 16900 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:37:59.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:58.437103 16891 server_base.cc:1034] running on GCE node
13:37:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.525957 16891 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:37:59.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.528959 16891 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:37:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.530371 16891 hybrid_clock.cc:648] HybridClock initialized: now 1714138679530327 us; error 59 us; skew 500 ppm
13:37:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.531121 16891 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:37:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:37:59 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138679, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:37:59.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.540501 16891 init.cc:377] Logged in from keytab as kudu/127.16.120.190@KRBTEST.COM (short username kudu)
13:37:59.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.545648 16891 webserver.cc:460] Webserver started at http://127.16.120.190:35425/ using document root <none> and password file <none>
13:37:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.546528 16891 fs_manager.cc:359] Metadata directory not provided
13:37:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.546751 16891 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:37:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.547220 16891 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:37:59.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.551864 16891 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/instance:
13:37:59.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b20ae5c389e34802834399d6011ed84f"
13:37:59.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:59 on dist-test-slave-6qgw"
13:37:59.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.552940 16891 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal/instance:
13:37:59.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b20ae5c389e34802834399d6011ed84f"
13:37:59.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:59 on dist-test-slave-6qgw"
13:37:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.559716 16891 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
13:37:59.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.565093 16908 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:37:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.566105 16891 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
13:37:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.566435 16891 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:37:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b20ae5c389e34802834399d6011ed84f"
13:37:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:59 on dist-test-slave-6qgw"
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.566771 16891 fs_report.cc:383] FS layout report
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/data
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:37:59.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.624089 16891 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:37:59.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.629637 16891 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:37:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.630112 16891 kserver.cc:163] Server-wide thread pool size limit: 3276
13:37:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.705961 16891 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.190:39275
13:37:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.706032 16959 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.190:39275 every 8 connection(s)
13:37:59.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.708706 16891 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:37:59.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.714185 16960 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:37:59.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.718256 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 16891
13:37:59.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.718693 16866 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal/instance
13:37:59.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.725026 16960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } has no permanent_uuid. Determining permanent_uuid...
13:37:59.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.120.189@KRBTEST.COM; defaulting to no policy
13:37:59.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:37:59.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.120.189@KRBTEST.COM" created.
13:37:59.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:37:59 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138679, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for kudu/127.16.120.190@KRBTEST.COM
13:37:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:37:59.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.120.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab.
13:37:59.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.120.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab.
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.761754 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/logs
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.189:38937
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.189
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.189
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:37:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:37:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:37:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:37:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:37:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:37:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:37:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:37:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:37:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.762439 16960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } has no permanent_uuid. Determining permanent_uuid...
13:37:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:59.764755 16911 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:37:59.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:59.769150 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.189:38937: Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:37:59.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.818567 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } attempt: 1
13:37:59.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:59.822186 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.189:38937: Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:37:59.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.890574 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } attempt: 2
13:37:59.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:59.894961 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.189:38937: Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:37:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:37:59.986413 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } attempt: 3
13:37:59.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:37:59.990222 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.189:38937: Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:38:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.051296 16967 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.051816 16967 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:00.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.052387 16967 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.080611 16967 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.080787 16967 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:00.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.081012 16967 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:00.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.081256 16967 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:00.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.081445 16967 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:00.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.081661 16967 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:00.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.081831 16967 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:00.116003 16967 master_runner.cc:386] Master server non-default flags:
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.189
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.189:38937
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.189
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/logs
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:00.117352 16967 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:00.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:00.118405 16967 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.131453 16974 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:00.133596 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } attempt: 4
13:38:00.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.138171 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.189:38937: Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:38:00.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:00.437536 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } attempt: 5
13:38:00.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.441073 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.189:38937: Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:38:00.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:00.988441 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } attempt: 6
13:38:00.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.994673 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.189:38937: Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:38:01.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.132011 16975 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:01.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:00.134413 16977 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:01.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.242504 16976 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:38:01.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.242543 16967 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:38:01.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.246634 16967 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.249089 16967 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.250444 16967 hybrid_clock.cc:648] HybridClock initialized: now 1714138681250395 us; error 61 us; skew 500 ppm
13:38:01.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.251173 16967 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:01 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138681, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.257098 16967 init.cc:377] Logged in from keytab as kudu/127.16.120.189@KRBTEST.COM (short username kudu)
13:38:01.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.262585 16967 webserver.cc:460] Webserver started at http://127.16.120.189:34767/ using document root <none> and password file <none>
13:38:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.263384 16967 fs_manager.cc:359] Metadata directory not provided
13:38:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.263552 16967 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:01.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.263908 16967 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:38:01.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.268365 16967 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/instance:
13:38:01.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:01.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:01 on dist-test-slave-6qgw"
13:38:01.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.269249 16967 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal/instance:
13:38:01.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:01.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:01 on dist-test-slave-6qgw"
13:38:01.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.276132 16967 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
13:38:01.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.281435 16987 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:01.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.282549 16967 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.282958 16967 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:01 on dist-test-slave-6qgw"
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.283233 16967 fs_report.cc:383] FS layout report
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/data
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:01.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:01.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:01.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:01.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.332278 16967 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.337885 16967 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.338330 16967 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.404634 16967 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.189:38937
13:38:01.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.404721 17038 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.189:38937 every 8 connection(s)
13:38:01.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.407179 16967 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.411927 17039 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:38:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.413653 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 16967
13:38:01.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.414040 16866 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal/instance
13:38:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.422922 17039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } has no permanent_uuid. Determining permanent_uuid...
13:38:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.120.188@KRBTEST.COM; defaulting to no policy
13:38:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:38:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.120.188@KRBTEST.COM" created.
13:38:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:01 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138681, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.189@KRBTEST.COM for kudu/127.16.120.190@KRBTEST.COM
13:38:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.454321 17039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } has no permanent_uuid. Determining permanent_uuid...
13:38:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:38:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.120.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab.
13:38:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.120.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab.
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.458806 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/logs
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.188:34525
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.188
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.188
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:01.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:01 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138681, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.189@KRBTEST.COM for kudu/127.16.120.189@KRBTEST.COM
13:38:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.476329 17039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } has no permanent_uuid. Determining permanent_uuid...
13:38:01.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.478847 16989 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.482859 17039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.532344 17039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 1
13:38:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.535990 17039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:01.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.604476 17039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 2
13:38:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.608248 17039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:01.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.699692 17039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 3
13:38:01.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.705210 17039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:01.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.757594 17046 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.758106 17046 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.758651 17046 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.787748 17046 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.787981 17046 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.788368 17046 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.788642 17046 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:01.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.788925 17046 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:01.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.789201 17046 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:01.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.789433 17046 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.824874 17046 master_runner.cc:386] Master server non-default flags:
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.188
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.188:34525
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.188
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/logs
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:01.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.826278 17046 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:01.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.827524 17046 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.843714 17054 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:01.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.848630 17039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 4
13:38:01.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.853057 17039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.055052 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } attempt: 7
13:38:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:02 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138679, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for kudu/127.16.120.189@KRBTEST.COM
13:38:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.077411 16960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } has no permanent_uuid. Determining permanent_uuid...
13:38:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:02.083851 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.142189 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 1
13:38:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:02.147019 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.152438 17039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 5
13:38:02.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:02.157255 17039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.228359 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 2
13:38:02.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:02.232324 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.317911 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 3
13:38:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:02.322280 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.462852 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 4
13:38:02.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:02.467290 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.704869 17039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 6
13:38:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:02.719754 17039 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.750867 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 5
13:38:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:02.755481 16960 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.16.120.188:34525: Network error: Client connection negotiation failed: client connection to 127.16.120.188:34525: connect: Connection refused (error 111)
13:38:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.844923 17056 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:01.845278 17046 server_base.cc:1034] running on GCE node
13:38:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:01.844352 17053 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:02.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.969537 17046 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:02.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.971928 17046 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:02.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.973286 17046 hybrid_clock.cc:648] HybridClock initialized: now 1714138682973232 us; error 66 us; skew 500 ppm
13:38:02.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.974277 17046 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:02 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138682, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.980382 17046 init.cc:377] Logged in from keytab as kudu/127.16.120.188@KRBTEST.COM (short username kudu)
13:38:02.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.985242 17046 webserver.cc:460] Webserver started at http://127.16.120.188:40037/ using document root <none> and password file <none>
13:38:02.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.986058 17046 fs_manager.cc:359] Metadata directory not provided
13:38:02.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.986249 17046 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:02.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.986706 17046 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:38:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.991173 17046 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/instance:
13:38:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "128c2ef67fa540439ad331f13119e135"
13:38:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:02 on dist-test-slave-6qgw"
13:38:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.992146 17046 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal/instance:
13:38:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "128c2ef67fa540439ad331f13119e135"
13:38:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:02 on dist-test-slave-6qgw"
13:38:02.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:02.999274 17046 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
13:38:03.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.004841 17069 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:03.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.005779 17046 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
13:38:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.006083 17046 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "128c2ef67fa540439ad331f13119e135"
13:38:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:02 on dist-test-slave-6qgw"
13:38:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.006418 17046 fs_report.cc:383] FS layout report
13:38:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/data
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:03.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.067833 17046 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.072742 17046 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.073211 17046 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.141590 17046 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.188:34525
13:38:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.141722 17120 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.188:34525 every 8 connection(s)
13:38:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.144136 17046 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:03.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.146471 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17046
13:38:03.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.147030 16866 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal/instance
13:38:03.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.150185 17121 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:38:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.162773 17121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } has no permanent_uuid. Determining permanent_uuid...
13:38:03.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138677, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.120.188@KRBTEST.COM
13:38:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138682, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.188@KRBTEST.COM for kudu/127.16.120.190@KRBTEST.COM
13:38:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.191411 17121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } has no permanent_uuid. Determining permanent_uuid...
13:38:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138682, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.188@KRBTEST.COM for kudu/127.16.120.189@KRBTEST.COM
13:38:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138677, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.120.189@KRBTEST.COM
13:38:03.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.217612 17121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } has no permanent_uuid. Determining permanent_uuid...
13:38:03.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138682, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.188@KRBTEST.COM for kudu/127.16.120.188@KRBTEST.COM
13:38:03.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138677, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.120.190@KRBTEST.COM
13:38:03.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.244026 17121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap starting.
13:38:03.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.244261 16866 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
13:38:03.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.248977 17121 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Neither blocks nor log segments found. Creating new log.
13:38:03.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.250890 17121 log.cc:824] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Log is configured to *not* fsync() on all Append() calls
13:38:03.251 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:03.251 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
13:38:03.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
13:38:03.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
13:38:03.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.120.190"
port: 39275
}
bound_http_address {
host: "127.16.120.190"
port: 35425
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.120.189"
port: 38937
}
bound_http_address {
host: "127.16.120.189"
port: 34767
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.120.188"
port: 34525
}
bound_http_address {
host: "127.16.120.188"
port: 40037
}
}
}
13:38:03.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:38:03.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:03.253 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:38:03.254 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:38:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.255748 17121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: No bootstrap required, opened a new log
13:38:03.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138683, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:03.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:38:03.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.284857 17121 raft_consensus.cc:348] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.285653 17121 raft_consensus.cc:374] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:38:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.285929 17121 raft_consensus.cc:729] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 128c2ef67fa540439ad331f13119e135, State: Initialized, Role: FOLLOWER
13:38:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.286732 17121 consensus_queue.cc:260] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.293 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:03.294 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc
13:38:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.294543 17133 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:03.295 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:38:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.295408 17133 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: FOLLOWER
13:38:03.295 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc
13:38:03.296 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:38:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.296204 17121 sys_catalog.cc:564] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: configured and running, proceeding with master startup.
13:38:03.297 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
13:38:03.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.307922 16960 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 6
13:38:03.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:03.322813 17145 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:38:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:03.323109 17145 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:38:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:03.328269 17075 tablet.cc:2369] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:38:03.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138679, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for kudu/127.16.120.188@KRBTEST.COM
13:38:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.346390 16960 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Bootstrap starting.
13:38:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.318475 17046 master_runner.cc:186] Error getting master registration for 127.16.120.190:39275: OK, instance_id { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" instance_seqno: 1714138679682444 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:38:03.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.350049 17046 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:38:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.351265 16960 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Neither blocks nor log segments found. Creating new log.
13:38:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.352797 16960 log.cc:824] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Log is configured to *not* fsync() on all Append() calls
13:38:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.356765 16960 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: No bootstrap required, opened a new log
13:38:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.375238 16960 raft_consensus.cc:348] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.375759 16960 raft_consensus.cc:374] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:38:03.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.376004 16960 raft_consensus.cc:729] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b20ae5c389e34802834399d6011ed84f, State: Initialized, Role: FOLLOWER
13:38:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.376775 16960 consensus_queue.cc:260] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.379199 17148 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:03.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.380003 17148 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.380846 16960 sys_catalog.cc:564] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: configured and running, proceeding with master startup.
13:38:03.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:03.388803 16914 tablet.cc:2369] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:38:03.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:03.408645 17159 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:38:03.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:03.409495 17159 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:38:03.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.416693 16891 master_runner.cc:186] Error getting master registration for 127.16.120.189:38937: OK, instance_id { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" instance_seqno: 1714138681385992 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:38:03.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.447844 16891 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:38:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.524737 17133 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:38:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.525249 17133 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.527343 17133 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.528721 16935 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b20ae5c389e34802834399d6011ed84f" is_pre_election: true
13:38:03.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.529649 16935 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 128c2ef67fa540439ad331f13119e135 in term 0.
13:38:03.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.530886 17072 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 128c2ef67fa540439ad331f13119e135, b20ae5c389e34802834399d6011ed84f; no voters:
13:38:03.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.531664 17133 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 0 FOLLOWER]: Leader pre-election won for term 1
13:38:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.531920 17133 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:38:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.532383 17133 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 0 FOLLOWER]: Advancing to term 1
13:38:03.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.538838 17133 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.540721 17133 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 1 election: Requested vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.542284 16935 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b20ae5c389e34802834399d6011ed84f"
13:38:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.542788 16935 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 0 FOLLOWER]: Advancing to term 1
13:38:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.549523 16935 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 128c2ef67fa540439ad331f13119e135 in term 1.
13:38:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.550521 17072 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 128c2ef67fa540439ad331f13119e135, b20ae5c389e34802834399d6011ed84f; no voters:
13:38:03.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.551306 17133 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Leader election won for term 1
13:38:03.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.558971 17133 raft_consensus.cc:686] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 LEADER]: Becoming Leader. State: Replica: 128c2ef67fa540439ad331f13119e135, State: Running, Role: LEADER
13:38:03.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.559908 17133 consensus_queue.cc:237] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.571460 17161 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 128c2ef67fa540439ad331f13119e135. Latest consensus state: current_term: 1 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:03.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.572284 17161 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: LEADER
13:38:03.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.575857 17163 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:38:03.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.582741 17163 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:38:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.601246 16935 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 1 FOLLOWER]: Refusing update from remote peer 128c2ef67fa540439ad331f13119e135: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:38:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.602757 17161 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:38:03.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.635179 17148 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 128c2ef67fa540439ad331f13119e135. Latest consensus state: current_term: 1 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:03.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.635751 17148 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.662868 17148 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.663645 17148 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.666190 17161 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.666934 17161 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: LEADER
13:38:03.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.669653 17163 catalog_manager.cc:1347] Generated new cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.669919 17163 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:38:03.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.694151 17163 catalog_manager.cc:1370] Generated new certificate authority record
13:38:03.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.696280 17163 catalog_manager.cc:1504] Loading token signing keys...
13:38:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.726399 17163 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Generated new TSK 0
13:38:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.730628 17163 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:38:03.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.780150 17039 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } attempt: 7
13:38:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:03 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138681, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.189@KRBTEST.COM for kudu/127.16.120.188@KRBTEST.COM
13:38:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.835306 17039 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap starting.
13:38:03.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.848590 17039 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Neither blocks nor log segments found. Creating new log.
13:38:03.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.852671 17039 log.cc:824] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Log is configured to *not* fsync() on all Append() calls
13:38:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.866559 17039 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: No bootstrap required, opened a new log
13:38:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.903126 17039 raft_consensus.cc:348] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.903967 17039 raft_consensus.cc:374] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:38:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.904256 17039 raft_consensus.cc:729] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84ff1fb2b572460e9fe8d2efeb514e46, State: Initialized, Role: FOLLOWER
13:38:03.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.905084 17039 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:03.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.913029 17171 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:03.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.913713 17171 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.916199 17039 sys_catalog.cc:564] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: configured and running, proceeding with master startup.
13:38:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.918102 17012 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 0 FOLLOWER]: Advancing to term 1
13:38:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.917035 17014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" is_pre_election: true
13:38:03.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.926857 17013 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:03.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.927706 17012 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 1 FOLLOWER]: Refusing update from remote peer 128c2ef67fa540439ad331f13119e135: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:38:03.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:03.930986 17133 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:38:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.000276 17171 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 128c2ef67fa540439ad331f13119e135. Latest consensus state: current_term: 1 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.000519 17177 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.001112 17171 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.001315 17177 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.062774 17188 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:04.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.064023 17188 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: loading cluster ID for follower catalog manager: success
13:38:04.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.071103 17188 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: acquiring CA information for follower catalog manager: success
13:38:04.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.076493 17188 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:04.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.382474 17133 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:04.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.383718 17133 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: LEADER
13:38:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.415095 17159 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.415767 17159 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: loading cluster ID for follower catalog manager: success
13:38:04.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.431509 17159 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: acquiring CA information for follower catalog manager: success
13:38:04.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:04.452143 17159 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:05.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:05.317659 16570 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0de0 after lost signal to thread 16442
13:38:05.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:05.318781 16570 debug-util.cc:398] Leaking SignalData structure 0x7b08000b01a0 after lost signal to thread 16573
13:38:06.298 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.188:34525
13:38:06.298 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
13:38:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:06.298861 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17046
13:38:06.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:06.350 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.189:38937
13:38:06.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
13:38:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:06.350827 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 16967
13:38:06.374 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:06.374 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.190:39275
13:38:06.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
13:38:06.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:06.375344 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 16891
13:38:06.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:06.417 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.188:34525
13:38:06.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
13:38:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:06.419049 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data
13:38:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/logs
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.188:34525
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.188
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40037
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.188
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.778174 17198 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.778755 17198 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.779434 17198 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:06.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.815598 17198 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.815834 17198 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.816246 17198 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.816548 17198 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.816875 17198 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.817090 17198 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.817236 17198 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:06.853276 17198 master_runner.cc:386] Master server non-default flags:
13:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data
13:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.188
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.188:34525
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:06.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.188
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40037
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/logs
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:06.854458 17198 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:06.855405 17198 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:06.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.872999 17206 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:08.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.873603 17205 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:08.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:06.874933 17208 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:08.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.221174 17207 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1347 milliseconds
13:38:08.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.221305 17198 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:38:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.222638 17198 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:08.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.225143 17198 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:08.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.226466 17198 hybrid_clock.cc:648] HybridClock initialized: now 1714138688226434 us; error 51 us; skew 500 ppm
13:38:08.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.227223 17198 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:08.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:08 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138688, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:08.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.238014 17198 init.cc:377] Logged in from keytab as kudu/127.16.120.188@KRBTEST.COM (short username kudu)
13:38:08.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.242655 17198 webserver.cc:460] Webserver started at http://127.16.120.188:40037/ using document root <none> and password file <none>
13:38:08.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.243454 17198 fs_manager.cc:359] Metadata directory not provided
13:38:08.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.243641 17198 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.250952 17198 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
13:38:08.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.255038 17217 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:08.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.256129 17198 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
13:38:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.256439 17198 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "128c2ef67fa540439ad331f13119e135"
13:38:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:02 on dist-test-slave-6qgw"
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.258678 17198 fs_report.cc:383] FS layout report
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/data
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:08.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.317044 17198 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.321960 17198 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.322369 17198 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:08.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.389626 17198 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.188:34525
13:38:08.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.389756 17268 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.188:34525 every 8 connection(s)
13:38:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.392270 17198 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:08.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.400911 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17198
13:38:08.401 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:08.401 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.189:38937
13:38:08.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.403090 17269 sys_catalog.cc:263] Verifying existing consensus state
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.403456 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/logs
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.189:38937
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.189
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34767
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.189
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.407838 17269 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap starting.
13:38:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.442628 17269 log.cc:824] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Log is configured to *not* fsync() on all Append() calls
13:38:08.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.456930 17269 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:38:08.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.457736 17269 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap complete.
13:38:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.481958 17269 raft_consensus.cc:348] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.484314 17269 raft_consensus.cc:729] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 128c2ef67fa540439ad331f13119e135, State: Initialized, Role: FOLLOWER
13:38:08.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.485195 17269 consensus_queue.cc:260] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.488077 17273 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:08.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.489034 17273 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: FOLLOWER
13:38:08.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.489679 17269 sys_catalog.cc:564] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: configured and running, proceeding with master startup.
13:38:08.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.523948 17284 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:08.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.524211 17284 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: loading cluster ID for follower catalog manager: success
13:38:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.532253 17284 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: acquiring CA information for follower catalog manager: success
13:38:08.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.537945 17284 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:08.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.513547 17198 master_runner.cc:186] Error getting master registration for 127.16.120.190:39275: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111),
13:38:08.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.539114 17198 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:38:08.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.757143 17271 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:08.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.757711 17271 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:08.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.758322 17271 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.788388 17271 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.788594 17271 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.788830 17271 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.789011 17271 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.789196 17271 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.789400 17271 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.789625 17271 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:08.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.826496 17271 master_runner.cc:386] Master server non-default flags:
13:38:08.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.189
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.189:38937
13:38:08.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.189
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34767
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/logs
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:08.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.827823 17271 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:08.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:08.829239 17271 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:08.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.848389 17293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:09.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:09.754025 17298 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:38:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:09.754961 17298 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:09.769248 17298 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:09.792390 17220 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:09.804952 17220 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:09.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:09.807520 17220 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937): Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:38:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:09.808005 17220 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 128c2ef67fa540439ad331f13119e135; no voters: 84ff1fb2b572460e9fe8d2efeb514e46, b20ae5c389e34802834399d6011ed84f
13:38:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:09.808827 17298 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
13:38:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:10.244930 17291 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 17271
13:38:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:10.427732 17291 kernel_stack_watchdog.cc:198] Thread 17271 stuck at /home/jenkins-slave/workspace/build_and_test@3/src/kudu/util/thread.cc:637 for 396ms:
13:38:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
13:38:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
13:38:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
13:38:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
13:38:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:08.849987 17292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:10.430410 17271 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.584s user 0.458s sys 0.974s
13:38:10.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:10.430666 17271 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.584s user 0.458s sys 0.974s
13:38:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:10.431732 17295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:10.434276 17294 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:38:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.434301 17271 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:38:10.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.435853 17271 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.439175 17271 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.440701 17271 hybrid_clock.cc:648] HybridClock initialized: now 1714138690440628 us; error 73 us; skew 500 ppm
13:38:10.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.441761 17271 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:10 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138690, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:10.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.448884 17271 init.cc:377] Logged in from keytab as kudu/127.16.120.189@KRBTEST.COM (short username kudu)
13:38:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.454135 17271 webserver.cc:460] Webserver started at http://127.16.120.189:34767/ using document root <none> and password file <none>
13:38:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.455353 17271 fs_manager.cc:359] Metadata directory not provided
13:38:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.455642 17271 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.465905 17271 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
13:38:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.471310 17308 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.472378 17271 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.472741 17271 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:01 on dist-test-slave-6qgw"
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.476033 17271 fs_report.cc:383] FS layout report
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/data
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:10.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.543221 17271 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.549176 17271 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.549628 17271 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.625528 17271 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.189:38937
13:38:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.625588 17359 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.189:38937 every 8 connection(s)
13:38:10.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.629266 17271 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:10.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.634876 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17271
13:38:10.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:10.635 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.190:39275
13:38:10.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.637357 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/logs
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.190:39275
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.190
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35425
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.190
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:10.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.642616 17360 sys_catalog.cc:263] Verifying existing consensus state
13:38:10.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.647732 17360 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap starting.
13:38:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.687623 17360 log.cc:824] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Log is configured to *not* fsync() on all Append() calls
13:38:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.727226 17360 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:38:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.728387 17360 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap complete.
13:38:10.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.767767 17360 raft_consensus.cc:348] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:10.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.771142 17360 raft_consensus.cc:729] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84ff1fb2b572460e9fe8d2efeb514e46, State: Initialized, Role: FOLLOWER
13:38:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.772243 17360 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.775200 17364 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.776320 17364 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:10.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.776965 17360 sys_catalog.cc:564] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: configured and running, proceeding with master startup.
13:38:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.822741 17375 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.823184 17375 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: loading cluster ID for follower catalog manager: success
13:38:10.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.840829 17375 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: acquiring CA information for follower catalog manager: success
13:38:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.817260 17271 master_runner.cc:186] Error getting master registration for 127.16.120.190:39275: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111),
13:38:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.850504 17271 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:38:10.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:10.852984 17375 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.086887 17362 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.087662 17362 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:11.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.088737 17362 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.140866 17362 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.141204 17362 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.141718 17362 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.142088 17362 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.142424 17362 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.142751 17362 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.143016 17362 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.203709 17362 master_runner.cc:386] Master server non-default flags:
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.190
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.190:39275
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.190
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35425
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/logs
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.205188 17362 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.206555 17362 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.226061 17384 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.355719 17389 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:38:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.356475 17389 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.361904 17389 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:11.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.410869 17220 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:11.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.422560 17335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" is_pre_election: true
13:38:11.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.423331 17335 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 128c2ef67fa540439ad331f13119e135 in term 1.
13:38:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.424778 17220 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 128c2ef67fa540439ad331f13119e135, 84ff1fb2b572460e9fe8d2efeb514e46; no voters: b20ae5c389e34802834399d6011ed84f
13:38:11.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.425794 17389 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Leader pre-election won for term 2
13:38:11.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.426120 17389 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:38:11.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.426470 17389 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 1 FOLLOWER]: Advancing to term 2
13:38:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.437891 17389 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:11.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.444826 17220 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:11.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.445901 17389 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 election: Requested vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:11.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.447317 17335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:11.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.447893 17335 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 1 FOLLOWER]: Advancing to term 2
13:38:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.460125 17335 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 128c2ef67fa540439ad331f13119e135 in term 2.
13:38:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.461057 17220 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 128c2ef67fa540439ad331f13119e135, 84ff1fb2b572460e9fe8d2efeb514e46; no voters: b20ae5c389e34802834399d6011ed84f
13:38:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.462333 17389 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Leader election won for term 2
13:38:11.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.470052 17389 raft_consensus.cc:686] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 LEADER]: Becoming Leader. State: Replica: 128c2ef67fa540439ad331f13119e135, State: Running, Role: LEADER
13:38:11.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.471192 17389 consensus_queue.cc:237] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.504390 17394 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 128c2ef67fa540439ad331f13119e135. Latest consensus state: current_term: 2 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.507083 17394 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: LEADER
13:38:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.522660 17396 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:38:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.531566 17396 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:38:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.535887 17396 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.536258 17396 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:38:11.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.541690 17396 catalog_manager.cc:1504] Loading token signing keys...
13:38:11.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.546123 17396 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Loaded TSK: 0
13:38:11.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.547597 17396 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:38:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.008565 17335 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Refusing update from remote peer 128c2ef67fa540439ad331f13119e135: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
13:38:12.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.014808 17394 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
13:38:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:12.017807 17220 consensus_peers.cc:485] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 -> Peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Couldn't send request to peer b20ae5c389e34802834399d6011ed84f. Status: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
13:38:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.033079 17399 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 128c2ef67fa540439ad331f13119e135. Latest consensus state: current_term: 2 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.033905 17399 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:12.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.041267 17394 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.043385 17394 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: LEADER
13:38:12.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.046360 17399 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:12.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.047060 17399 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.226992 17383 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:11.238296 17386 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:11.235455 17362 server_base.cc:1034] running on GCE node
13:38:12.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.521337 17362 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:12.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.524514 17362 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.525971 17362 hybrid_clock.cc:648] HybridClock initialized: now 1714138692525877 us; error 97 us; skew 500 ppm
13:38:12.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.526858 17362 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:12.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:12 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138692, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.533250 17362 init.cc:377] Logged in from keytab as kudu/127.16.120.190@KRBTEST.COM (short username kudu)
13:38:12.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.539230 17362 webserver.cc:460] Webserver started at http://127.16.120.190:35425/ using document root <none> and password file <none>
13:38:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.540424 17362 fs_manager.cc:359] Metadata directory not provided
13:38:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.540735 17362 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.551255 17362 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.004s
13:38:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.556655 17409 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:12.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.557850 17362 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
13:38:12.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.558279 17362 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:12.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b20ae5c389e34802834399d6011ed84f"
13:38:12.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:59 on dist-test-slave-6qgw"
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.561756 17362 fs_report.cc:383] FS layout report
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/data
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:12.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:12.605216 16293 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad280 after lost signal to thread 16228
13:38:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:12.606031 16293 debug-util.cc:398] Leaking SignalData structure 0x7b0800083940 after lost signal to thread 16296
13:38:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.688647 17362 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.704630 17362 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.705207 17362 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.848771 17362 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.190:39275
13:38:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.848951 17460 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.190:39275 every 8 connection(s)
13:38:12.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.852385 17362 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.857035 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17362
13:38:12.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:12.858 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.120.188:34525(127.16.120.188:34525)] connecting to peer
13:38:12.859 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.120.189:38937(127.16.120.189:38937)] connecting to peer
13:38:12.863 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.120.190:39275(127.16.120.190:39275)] connecting to peer
13:38:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.871263 17461 sys_catalog.cc:263] Verifying existing consensus state
13:38:12.875 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.120.188:34525(127.16.120.188:34525)] Successfully connected to peer
13:38:12.877 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.120.190:39275(127.16.120.190:39275)] Successfully connected to peer
13:38:12.891 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.120.189:38937(127.16.120.189:38937)] Successfully connected to peer
13:38:12.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.901862 17461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Bootstrap starting.
13:38:12.903 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
13:38:12.918 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.16.120.188
13:38:12.923 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.16.120.189
13:38:12.923 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.16.120.190
13:38:12.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:12.979710 17461 log.cc:824] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Log is configured to *not* fsync() on all Append() calls
13:38:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:12 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138683, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.120.188@KRBTEST.COM
13:38:13.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.120.189:38937(127.16.120.189:38937)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
13:38:13.007 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.120.189:38937: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:13.012329 17468 negotiation.cc:337] Failed RPC negotiation. Trace:
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.891968 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:32978 (local address 127.16.120.189:38937)
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.899849 (+ 7881us) server_negotiation.cc:206] Beginning negotiation
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.899873 (+ 24us) server_negotiation.cc:399] Waiting for connection header
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.899919 (+ 46us) server_negotiation.cc:407] Connection header received
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.900053 (+ 134us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.900078 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.900368 (+ 290us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.900596 (+ 228us) server_negotiation.cc:226] Negotiated authn=SASL
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.934377 (+ 33781us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.936388 (+ 2011us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.995823 (+ 59435us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
13:38:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.996549 (+ 726us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
13:38:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:13.010702 (+ 14153us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:32978: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:32978): Cannot send after transport endpoint shutdown (error 108)
13:38:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7248,"thread_start_us":6975,"threads_started":1}
13:38:13.013 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.120.189:38937(127.16.120.189:38937)] handling channelInactive
13:38:13.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.028241 17461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:38:13.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.120.190:39275(127.16.120.190:39275)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
13:38:13.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.120.190:39275: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
13:38:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.036751 17461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Bootstrap complete.
13:38:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:13.036772 17467 negotiation.cc:337] Failed RPC negotiation. Trace:
13:38:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.876485 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41696 (local address 127.16.120.190:39275)
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.897628 (+ 21143us) server_negotiation.cc:206] Beginning negotiation
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.897670 (+ 42us) server_negotiation.cc:399] Waiting for connection header
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.897742 (+ 72us) server_negotiation.cc:407] Connection header received
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.914274 (+ 16532us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.914340 (+ 66us) server_negotiation.cc:461] Received NEGOTIATE request from client
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.914831 (+ 491us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.915286 (+ 455us) server_negotiation.cc:226] Negotiated authn=SASL
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.926847 (+ 11561us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:12.936141 (+ 9294us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:13.029507 (+ 93366us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:13.030331 (+ 824us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:13.033738 (+ 3407us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41696: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:41696): Cannot send after transport endpoint shutdown (error 108)
13:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":20331,"thread_start_us":20029,"threads_started":1}
13:38:13.039 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.120.190:39275(127.16.120.190:39275)] handling channelInactive
13:38:13.042 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf9698a9f, L:/127.0.0.1:47822 - R:/127.16.120.188:34525] using SASL/GSSAPI
13:38:13.055 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
13:38:13.057 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=198), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.120.188:34525, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.120.189:38937, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.120.190:39275, [148ms] Sub RPC ConnectToMaster: received response from server master-127.16.120.189:38937: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [174ms] Sub RPC ConnectToMaster: received response from server master-127.16.120.190:39275: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [196ms] Sub RPC ConnectToMaster: received response from server master-127.16.120.188:34525: OK, deferred=null) after lookup
13:38:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:13 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138688, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.188@KRBTEST.COM for kudu/127.16.120.190@KRBTEST.COM
13:38:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.068554 17461 raft_consensus.cc:348] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.071987 17461 raft_consensus.cc:729] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b20ae5c389e34802834399d6011ed84f, State: Initialized, Role: FOLLOWER
13:38:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.072891 17461 consensus_queue.cc:260] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:13.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.080956 17472 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:13.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.081894 17472 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:13.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.084529 17461 sys_catalog.cc:564] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: configured and running, proceeding with master startup.
13:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.090248 17436 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 1 FOLLOWER]: Advancing to term 2
13:38:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.147078 17472 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 128c2ef67fa540439ad331f13119e135. Latest consensus state: current_term: 2 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.147873 17472 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.148725 17480 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.149504 17480 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:13.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.176129 17486 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:13.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.176390 17486 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: loading cluster ID for follower catalog manager: success
13:38:13.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:13 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138692, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for kudu/127.16.120.190@KRBTEST.COM
13:38:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.191354 17486 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: acquiring CA information for follower catalog manager: success
13:38:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.201653 17486 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:13.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:13.400343 16372 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2540 after lost signal to thread 16304
13:38:13.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:13.400939 16372 debug-util.cc:398] Leaking SignalData structure 0x7b08000b77a0 after lost signal to thread 16375
13:38:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.592058 17490 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:13.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:13.593173 17490 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: LEADER
13:38:16.116 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.188:34525
13:38:16.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
13:38:16.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:16.116731 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17198
13:38:16.145 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.120.188:34525(127.16.120.188:34525)] handling channelInactive
13:38:16.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:16.145 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.189:38937
13:38:16.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.120.188:34525(127.16.120.188:34525)] cleaning up while in state READY due to: connection closed
13:38:16.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
13:38:16.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:16.146175 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17271
13:38:16.172 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:16.172 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.190:39275
13:38:16.172 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
13:38:16.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:16.173058 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17362
13:38:16.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:16.199 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.188:34525
13:38:16.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
13:38:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:16.201426 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data
13:38:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:16.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/logs
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.188:34525
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.188
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40037
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.188
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:16.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.497627 17501 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:16.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.498275 17501 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.498965 17501 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.528048 17501 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.528292 17501 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:16.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.528676 17501 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.528944 17501 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.529246 17501 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.529551 17501 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.529753 17501 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:16.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:16.563472 17501 master_runner.cc:386] Master server non-default flags:
13:38:16.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:16.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:16.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.188
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.188:34525
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.188
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40037
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/logs
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:16.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:16.564853 17501 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:16.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:16.566248 17501 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:16.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.581714 17507 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.582432 17510 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:16.583089 17501 server_base.cc:1034] running on GCE node
13:38:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:16.585511 17508 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.664291 17501 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:17.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.666806 17501 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:17.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.668136 17501 hybrid_clock.cc:648] HybridClock initialized: now 1714138697668106 us; error 53 us; skew 500 ppm
13:38:17.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.668884 17501 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:17 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138697, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.680029 17501 init.cc:377] Logged in from keytab as kudu/127.16.120.188@KRBTEST.COM (short username kudu)
13:38:17.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.685729 17501 webserver.cc:460] Webserver started at http://127.16.120.188:40037/ using document root <none> and password file <none>
13:38:17.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.686808 17501 fs_manager.cc:359] Metadata directory not provided
13:38:17.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.687034 17501 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:17.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.697119 17501 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
13:38:17.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.702690 17520 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:17.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.703814 17501 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
13:38:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.704131 17501 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "128c2ef67fa540439ad331f13119e135"
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:02 on dist-test-slave-6qgw"
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.706959 17501 fs_report.cc:383] FS layout report
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/data
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:17.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.763868 17501 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.769264 17501 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.769717 17501 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.842455 17501 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.188:34525
13:38:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.842569 17571 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.188:34525 every 8 connection(s)
13:38:17.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.845218 17501 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.853857 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17501
13:38:17.854 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:17.854 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.189:38937
13:38:17.854 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.856134 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/logs
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.189:38937
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.189
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34767
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.189
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:17.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.857841 17572 sys_catalog.cc:263] Verifying existing consensus state
13:38:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.864454 17572 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap starting.
13:38:17.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.899425 17572 log.cc:824] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Log is configured to *not* fsync() on all Append() calls
13:38:17.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.914665 17572 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:38:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.915416 17572 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap complete.
13:38:17.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.938926 17572 raft_consensus.cc:348] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:17.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.941671 17572 raft_consensus.cc:729] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 128c2ef67fa540439ad331f13119e135, State: Initialized, Role: FOLLOWER
13:38:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.942564 17572 consensus_queue.cc:260] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:17.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.945504 17576 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:17.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.946517 17576 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: FOLLOWER
13:38:17.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.947211 17572 sys_catalog.cc:564] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: configured and running, proceeding with master startup.
13:38:17.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.975220 17587 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:17.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.975706 17587 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: loading cluster ID for follower catalog manager: success
13:38:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.984855 17587 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: acquiring CA information for follower catalog manager: success
13:38:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.990466 17587 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:17.971973 17501 master_runner.cc:186] Error getting master registration for 127.16.120.190:39275: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111),
13:38:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:18.001667 17501 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:38:18.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.183631 17574 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:18.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.184233 17574 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:18.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.184870 17574 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:18.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.215145 17574 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:18.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.215385 17574 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:18.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.215751 17574 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:18.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.216006 17574 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:18.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.216272 17574 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:18.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.216523 17574 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:18.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.216722 17574 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:18.251888 17574 master_runner.cc:386] Master server non-default flags:
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.189
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.189:38937
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.189
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34767
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/logs
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:18.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:18.253276 17574 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:18.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:18.254519 17574 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:18.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.271560 17598 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.264294 17602 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:38:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.264878 17602 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:19.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.267815 17602 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:19.271561 17523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:19.276890 17523 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:19.277545 17523 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937): Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:38:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.277920 17523 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 128c2ef67fa540439ad331f13119e135; no voters: 84ff1fb2b572460e9fe8d2efeb514e46, b20ae5c389e34802834399d6011ed84f
13:38:19.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.278682 17602 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
13:38:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:18.273793 17574 server_base.cc:1034] running on GCE node
13:38:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.273259 17595 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:18.271989 17596 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.420995 17574 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.424000 17574 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.425431 17574 hybrid_clock.cc:648] HybridClock initialized: now 1714138699425370 us; error 69 us; skew 500 ppm
13:38:19.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.426251 17574 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:19.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:19 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138699, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:19.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.433234 17574 init.cc:377] Logged in from keytab as kudu/127.16.120.189@KRBTEST.COM (short username kudu)
13:38:19.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.439106 17574 webserver.cc:460] Webserver started at http://127.16.120.189:34767/ using document root <none> and password file <none>
13:38:19.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.440340 17574 fs_manager.cc:359] Metadata directory not provided
13:38:19.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.440646 17574 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.452111 17574 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
13:38:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.457660 17610 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:19.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.458763 17574 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
13:38:19.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.459132 17574 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:19.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:19.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:01 on dist-test-slave-6qgw"
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.462401 17574 fs_report.cc:383] FS layout report
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/data
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:19.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.529677 17574 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.535449 17574 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.535928 17574 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.628181 17574 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.189:38937
13:38:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.628376 17661 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.189:38937 every 8 connection(s)
13:38:19.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.631217 17574 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.637892 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17574
13:38:19.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:19.638 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.190:39275
13:38:19.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
13:38:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.640445 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/logs
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.190:39275
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.190
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35425
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.190
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.646386 17662 sys_catalog.cc:263] Verifying existing consensus state
13:38:19.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.654011 17662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap starting.
13:38:19.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.705502 17662 log.cc:824] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Log is configured to *not* fsync() on all Append() calls
13:38:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.746567 17662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:38:19.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.747499 17662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap complete.
13:38:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.783255 17662 raft_consensus.cc:348] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:19.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.786537 17662 raft_consensus.cc:729] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84ff1fb2b572460e9fe8d2efeb514e46, State: Initialized, Role: FOLLOWER
13:38:19.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.788367 17662 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:19.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.791250 17666 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:19.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.792234 17666 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:19.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.792820 17662 sys_catalog.cc:564] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: configured and running, proceeding with master startup.
13:38:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.849602 17677 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.852054 17677 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: loading cluster ID for follower catalog manager: success
13:38:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.864434 17677 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: acquiring CA information for follower catalog manager: success
13:38:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.832257 17574 master_runner.cc:186] Error getting master registration for 127.16.120.190:39275: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111),
13:38:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.866896 17574 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:38:19.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:19.872804 17677 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:20.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.066313 17664 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:20.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.066955 17664 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:20.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.067538 17664 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.098677 17664 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.099004 17664 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.099503 17664 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.099830 17664 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.100207 17664 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.100579 17664 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:20.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.100858 17664 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:20.142954 17664 master_runner.cc:386] Master server non-default flags:
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.190
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.190:39275
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.190
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35425
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/logs
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:20.144011 17664 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:20.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:20.145041 17664 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.162092 17686 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.163484 17688 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:20.164763 17664 server_base.cc:1034] running on GCE node
13:38:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:20.165163 17685 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:21.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.339716 17664 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:21.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.342768 17664 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.344205 17664 hybrid_clock.cc:648] HybridClock initialized: now 1714138701344160 us; error 46 us; skew 500 ppm
13:38:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.344940 17664 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:21.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:21 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138701, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:21.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.350715 17664 init.cc:377] Logged in from keytab as kudu/127.16.120.190@KRBTEST.COM (short username kudu)
13:38:21.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.356344 17664 webserver.cc:460] Webserver started at http://127.16.120.190:35425/ using document root <none> and password file <none>
13:38:21.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.357292 17664 fs_manager.cc:359] Metadata directory not provided
13:38:21.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.357542 17664 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:21.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.365545 17664 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
13:38:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.370174 17697 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.371196 17664 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
13:38:21.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.371498 17664 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:21.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b20ae5c389e34802834399d6011ed84f"
13:38:21.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:59 on dist-test-slave-6qgw"
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.373647 17664 fs_report.cc:383] FS layout report
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/data
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.413578 17701 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:38:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.414378 17701 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.417037 17701 leader_election.cc:288] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 128c2ef67fa540439ad331f13119e135 (127.16.120.188:34525)
13:38:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:21.425153 17613 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:21.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.455682 17664 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.475953 17664 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:21.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.476557 17664 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.487316 17708 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:38:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.488678 17708 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:21.489495 17613 leader_election.cc:334] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:21.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.494268 17708 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:21.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.511629 17547 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "128c2ef67fa540439ad331f13119e135" is_pre_election: true
13:38:21.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.512470 17547 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84ff1fb2b572460e9fe8d2efeb514e46 in term 2.
13:38:21.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.514185 17612 leader_election.cc:302] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 128c2ef67fa540439ad331f13119e135, 84ff1fb2b572460e9fe8d2efeb514e46; no voters: b20ae5c389e34802834399d6011ed84f
13:38:21.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.515609 17701 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Leader pre-election won for term 3
13:38:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.515947 17701 raft_consensus.cc:482] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:38:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.516291 17701 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 2 FOLLOWER]: Advancing to term 3
13:38:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.526437 17701 raft_consensus.cc:504] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:21.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.533056 17547 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "128c2ef67fa540439ad331f13119e135"
13:38:21.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.533756 17547 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 2 FOLLOWER]: Advancing to term 3
13:38:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:21.537137 17613 leader_election.cc:334] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:21.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:21.543666 17523 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:21.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.544215 17701 leader_election.cc:288] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [CANDIDATE]: Term 3 election: Requested vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 128c2ef67fa540439ad331f13119e135 (127.16.120.188:34525)
13:38:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.546247 17547 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84ff1fb2b572460e9fe8d2efeb514e46 in term 3.
13:38:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.547389 17612 leader_election.cc:302] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 128c2ef67fa540439ad331f13119e135, 84ff1fb2b572460e9fe8d2efeb514e46; no voters: b20ae5c389e34802834399d6011ed84f
13:38:21.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.548570 17701 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 3 FOLLOWER]: Leader election won for term 3
13:38:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.555369 17701 raft_consensus.cc:686] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 3 LEADER]: Becoming Leader. State: Replica: 84ff1fb2b572460e9fe8d2efeb514e46, State: Running, Role: LEADER
13:38:21.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.556528 17701 consensus_queue.cc:237] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:21.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.569783 17637 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" is_pre_election: true
13:38:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.572619 17523 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 128c2ef67fa540439ad331f13119e135; no voters: 84ff1fb2b572460e9fe8d2efeb514e46, b20ae5c389e34802834399d6011ed84f
13:38:21.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.573812 17708 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
13:38:21.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.573597 17714 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 84ff1fb2b572460e9fe8d2efeb514e46. Latest consensus state: current_term: 3 leader_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:21.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.574460 17714 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: LEADER
13:38:21.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.577212 17716 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:38:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.586051 17716 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:38:21.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.590775 17716 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:21.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.591106 17716 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:38:21.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.596333 17716 catalog_manager.cc:1504] Loading token signing keys...
13:38:21.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.600802 17716 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Loaded TSK: 0
13:38:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.602330 17716 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:38:21.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.652372 17664 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.190:39275
13:38:21.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.652505 17761 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.190:39275 every 8 connection(s)
13:38:21.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.656519 17664 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:21.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.664065 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17664
13:38:21.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:21.665 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.120.188:34525(127.16.120.188:34525)] connecting to peer
13:38:21.667 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.120.188:34525(127.16.120.188:34525)] Successfully connected to peer
13:38:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.682001 17762 sys_catalog.cc:263] Verifying existing consensus state
13:38:21.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.690479 17762 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Bootstrap starting.
13:38:21.699 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd8dcc20d, L:/127.0.0.1:47860 - R:/127.16.120.188:34525] using TOKEN/null
13:38:21.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.738588 17762 log.cc:824] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Log is configured to *not* fsync() on all Append() calls
13:38:21.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.753942 17762 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:38:21.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.754614 17762 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Bootstrap complete.
13:38:21.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.777397 17762 raft_consensus.cc:348] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.779808 17762 raft_consensus.cc:729] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: b20ae5c389e34802834399d6011ed84f, State: Initialized, Role: FOLLOWER
13:38:21.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.780529 17762 consensus_queue.cc:260] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:21.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.783329 17765 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:21.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.784102 17765 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.784884 17762 sys_catalog.cc:564] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: configured and running, proceeding with master startup.
13:38:21.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.825232 17776 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:21.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.825501 17776 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: loading cluster ID for follower catalog manager: success
13:38:21.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:21 dist-test-slave-6qgw krb5kdc[16877](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138701, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for kudu/127.16.120.190@KRBTEST.COM
13:38:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.833484 17776 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: acquiring CA information for follower catalog manager: success
13:38:21.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.839215 17776 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:21.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.938422 17547 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Refusing update from remote peer 84ff1fb2b572460e9fe8d2efeb514e46: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
13:38:21.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.939849 17714 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [LEADER]: Connected to new peer: Peer: permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
13:38:21.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.946916 17708 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 84ff1fb2b572460e9fe8d2efeb514e46. Latest consensus state: current_term: 3 leader_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:21.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.947647 17708 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: FOLLOWER
13:38:21.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.949903 17701 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:21.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.951364 17701 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: LEADER
13:38:21.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.952883 17708 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:21.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.953497 17708 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: FOLLOWER
13:38:21.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.960508 17737 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 2 FOLLOWER]: Advancing to term 3
13:38:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.970068 17737 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [term 3 FOLLOWER]: Refusing update from remote peer 84ff1fb2b572460e9fe8d2efeb514e46: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
13:38:21.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.971551 17701 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
13:38:21.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.980100 17785 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:21.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.981134 17785 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:21.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.984527 17765 sys_catalog.cc:455] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 84ff1fb2b572460e9fe8d2efeb514e46. Latest consensus state: current_term: 3 leader_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:21.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:21.985073 17765 sys_catalog.cc:458] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f [sys.catalog]: This master's current role is: FOLLOWER
13:38:22.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:22.163600 17568 debug-util.cc:398] Leaking SignalData structure 0x7b080008ee60 after lost signal to thread 17502
13:38:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:22.465806 17701 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:22.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:22.469408 17701 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: LEADER
13:38:22.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:22.515229 16711 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0f80 after lost signal to thread 16585
13:38:22.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:22.516469 16711 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd580 after lost signal to thread 16714
13:38:23.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:23.580672 16218 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc840 after lost signal to thread 16155
13:38:23.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:23.581425 16218 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9e60 after lost signal to thread 16221
13:38:24.751 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.188:34525
13:38:24.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
13:38:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:24.752166 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17501
13:38:24.796 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.120.188:34525(127.16.120.188:34525)] handling channelInactive
13:38:24.796 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.120.188:34525(127.16.120.188:34525)] cleaning up while in state READY due to: connection closed
13:38:24.798 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:24.798 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.189:38937
13:38:24.798 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
13:38:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:24.798821 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17574
13:38:24.848 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:24.848 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.120.190:39275
13:38:24.848 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
13:38:24.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:24.849013 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17664
13:38:24.886 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:24.887 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.188:34525
13:38:24.887 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:24.888942 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/logs
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.188:34525
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.188
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40037
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.188
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:25.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.237929 17799 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:25.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.238606 17799 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:25.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.239272 17799 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.272811 17799 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.273027 17799 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.273314 17799 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.273561 17799 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.273764 17799 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.273978 17799 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.274125 17799 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:25.308821 17799 master_runner.cc:386] Master server non-default flags:
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.188.keytab
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.188
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.188:34525
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.188
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40037
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/logs
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:25.309967 17799 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:25.310858 17799 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:25.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.329597 17806 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.335021 17809 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:26.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:25.330782 17807 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:26.557987 17808 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1225 milliseconds
13:38:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.558111 17799 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:38:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.559405 17799 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.562280 17799 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.563711 17799 hybrid_clock.cc:648] HybridClock initialized: now 1714138706563668 us; error 72 us; skew 500 ppm
13:38:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.564464 17799 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:26.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:26 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138706, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:26.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.574301 17799 init.cc:377] Logged in from keytab as kudu/127.16.120.188@KRBTEST.COM (short username kudu)
13:38:26.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.579232 17799 webserver.cc:460] Webserver started at http://127.16.120.188:40037/ using document root <none> and password file <none>
13:38:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.580052 17799 fs_manager.cc:359] Metadata directory not provided
13:38:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.580229 17799 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:26.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.587862 17799 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
13:38:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.591863 17818 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.592748 17799 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.593086 17799 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "128c2ef67fa540439ad331f13119e135"
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:02 on dist-test-slave-6qgw"
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.595392 17799 fs_report.cc:383] FS layout report
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/wal
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/data
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:26.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:26.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.653982 17799 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.659502 17799 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.659938 17799 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:26.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.730968 17799 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.188:34525
13:38:26.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.731132 17869 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.188:34525 every 8 connection(s)
13:38:26.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.733597 17799 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-2/data/info.pb
13:38:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.738777 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17799
13:38:26.739 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:26.739 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.189:38937
13:38:26.739 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.740798 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/logs
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.189:38937
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.189
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34767
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.189
13:38:26.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.744047 17870 sys_catalog.cc:263] Verifying existing consensus state
13:38:26.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.751557 17870 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap starting.
13:38:26.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.800747 17870 log.cc:824] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Log is configured to *not* fsync() on all Append() calls
13:38:26.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.821144 17870 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:38:26.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.822093 17870 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Bootstrap complete.
13:38:26.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.857939 17870 raft_consensus.cc:348] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:26.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.861918 17870 raft_consensus.cc:729] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 128c2ef67fa540439ad331f13119e135, State: Initialized, Role: FOLLOWER
13:38:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.862874 17870 consensus_queue.cc:260] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:26.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.865645 17875 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.866778 17875 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: FOLLOWER
13:38:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.867720 17870 sys_catalog.cc:564] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: configured and running, proceeding with master startup.
13:38:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.898893 17886 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.899423 17886 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: loading cluster ID for follower catalog manager: success
13:38:26.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.906942 17886 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: acquiring CA information for follower catalog manager: success
13:38:26.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.912253 17886 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.895697 17799 master_runner.cc:186] Error getting master registration for 127.16.120.190:39275: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111),
13:38:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:26.931838 17799 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:38:27.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.117447 17872 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:27.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.118180 17872 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:27.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.119169 17872 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:27.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.154485 17872 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:27.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.154733 17872 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.155130 17872 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.155401 17872 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.155683 17872 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:27.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.155959 17872 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:27.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.156168 17872 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:27.194696 17872 master_runner.cc:386] Master server non-default flags:
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.189.keytab
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.189
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.189:38937
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.189
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34767
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/logs
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:27.196118 17872 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:27.197355 17872 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:27.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.214259 17895 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.175943 17901 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:38:28.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.176411 17901 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:28.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.179373 17901 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:28.185633 17821 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:28.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:28.190310 17821 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:28.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:28.191186 17821 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937): Network error: Client connection negotiation failed: client connection to 127.16.120.189:38937: connect: Connection refused (error 111)
13:38:28.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.191605 17821 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 128c2ef67fa540439ad331f13119e135; no voters: 84ff1fb2b572460e9fe8d2efeb514e46, b20ae5c389e34802834399d6011ed84f
13:38:28.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.192256 17901 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
13:38:28.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.214857 17897 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:28.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:27.217087 17872 server_base.cc:1034] running on GCE node
13:38:28.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:27.215564 17894 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:28.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.434075 17872 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:28.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.437963 17872 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:28.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.439327 17872 hybrid_clock.cc:648] HybridClock initialized: now 1714138708439267 us; error 76 us; skew 500 ppm
13:38:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.440352 17872 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:28.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:28 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138708, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.445792 17872 init.cc:377] Logged in from keytab as kudu/127.16.120.189@KRBTEST.COM (short username kudu)
13:38:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.451735 17872 webserver.cc:460] Webserver started at http://127.16.120.189:34767/ using document root <none> and password file <none>
13:38:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.452497 17872 fs_manager.cc:359] Metadata directory not provided
13:38:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.452677 17872 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.460944 17872 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.004s
13:38:28.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.465770 17910 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:28.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.466722 17872 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.467031 17872 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:38:01 on dist-test-slave-6qgw"
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.469220 17872 fs_report.cc:383] FS layout report
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/wal
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/data
13:38:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.530552 17872 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:28.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.537832 17872 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:28.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.538424 17872 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.628687 17872 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.189:38937
13:38:28.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.629294 17961 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.189:38937 every 8 connection(s)
13:38:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.632436 17872 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-1/data/info.pb
13:38:28.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.639659 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17872
13:38:28.640 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:28.640 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.120.190:39275
13:38:28.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
13:38:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.642671 16866 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/logs
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:28.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.190:39275
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.190
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35425
13:38:28.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.190
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:38:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.648736 17962 sys_catalog.cc:263] Verifying existing consensus state
13:38:28.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.656404 17962 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap starting.
13:38:28.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.714090 17962 log.cc:824] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Log is configured to *not* fsync() on all Append() calls
13:38:28.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.737485 17962 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:38:28.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.738420 17962 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: Bootstrap complete.
13:38:28.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.765259 17962 raft_consensus.cc:348] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.767730 17962 raft_consensus.cc:729] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84ff1fb2b572460e9fe8d2efeb514e46, State: Initialized, Role: FOLLOWER
13:38:28.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.768774 17962 consensus_queue.cc:260] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:28.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.771660 17966 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:28.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.772702 17966 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.773442 17962 sys_catalog.cc:564] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: configured and running, proceeding with master startup.
13:38:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.816319 17977 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.816762 17977 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: loading cluster ID for follower catalog manager: success
13:38:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.825739 17977 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: acquiring CA information for follower catalog manager: success
13:38:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.836918 17977 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:38:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.811530 17872 master_runner.cc:186] Error getting master registration for 127.16.120.190:39275: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111),
13:38:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:28.846690 17872 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:38:29.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.026619 17964 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:38:29.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.027108 17964 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:38:29.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.027768 17964 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:38:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.057063 17964 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:38:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.057318 17964 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:38:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.057694 17964 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:38:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.057952 17964 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:38:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.058195 17964 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:38:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.058439 17964 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:38:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.058632 17964 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38515/jwks.json
13:38:29.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.077127 16851 debug-util.cc:398] Leaking SignalData structure 0x7b08000b00c0 after lost signal to thread 16726
13:38:29.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.078245 16851 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd580 after lost signal to thread 16854
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.098843 17964 master_runner.cc:386] Master server non-default flags:
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.120.148:39455
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.120.190:39275,127.16.120.189:38937,127.16.120.188:34525
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/krb5kdc/kudu_127.16.120.190.keytab
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.120.190
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.120.190:39275
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38515/jwks.json
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:38:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.120.190
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35425
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/logs
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:38:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.100378 17964 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:38:29.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.101676 17964 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:38:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.155440 17985 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:29.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.708921 17991 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:38:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.709789 17991 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:29.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.714921 17991 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.762948 17821 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:29.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.775491 17937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" is_pre_election: true
13:38:29.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.777031 17937 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 128c2ef67fa540439ad331f13119e135 in term 3.
13:38:29.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.780514 17821 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 128c2ef67fa540439ad331f13119e135, 84ff1fb2b572460e9fe8d2efeb514e46; no voters: b20ae5c389e34802834399d6011ed84f
13:38:29.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.788599 17991 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Leader pre-election won for term 4
13:38:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.789105 17991 raft_consensus.cc:482] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:38:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.789582 17991 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 3 FOLLOWER]: Advancing to term 4
13:38:29.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.807979 17991 raft_consensus.cc:504] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:29.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.817200 17821 leader_election.cc:334] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111)
13:38:29.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.820338 17937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "128c2ef67fa540439ad331f13119e135" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "84ff1fb2b572460e9fe8d2efeb514e46"
13:38:29.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.821228 17937 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 3 FOLLOWER]: Advancing to term 4
13:38:29.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.835839 17991 leader_election.cc:288] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 election: Requested vote from peers b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275), 84ff1fb2b572460e9fe8d2efeb514e46 (127.16.120.189:38937)
13:38:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.843008 17937 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 128c2ef67fa540439ad331f13119e135 in term 4.
13:38:29.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.853595 17821 leader_election.cc:302] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 128c2ef67fa540439ad331f13119e135, 84ff1fb2b572460e9fe8d2efeb514e46; no voters: b20ae5c389e34802834399d6011ed84f
13:38:29.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.855428 17991 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 4 FOLLOWER]: Leader election won for term 4
13:38:29.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.865885 17991 raft_consensus.cc:686] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [term 4 LEADER]: Becoming Leader. State: Replica: 128c2ef67fa540439ad331f13119e135, State: Running, Role: LEADER
13:38:29.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.868719 17991 consensus_queue.cc:237] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } }
13:38:29.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.897805 17996 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 128c2ef67fa540439ad331f13119e135. Latest consensus state: current_term: 4 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:29.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.898836 17996 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: LEADER
13:38:29.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.931841 17998 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:38:29.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.939641 17998 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:38:29.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.943363 17998 catalog_manager.cc:1259] Loaded cluster ID: c73f31c8a8eb4439a56bfadc3a97f303
13:38:29.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.943589 17998 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:38:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.947600 17998 catalog_manager.cc:1504] Loading token signing keys...
13:38:29.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.951241 17998 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135: Loaded TSK: 0
13:38:29.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.952330 17998 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:38:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.258349 17937 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [term 4 FOLLOWER]: Refusing update from remote peer 128c2ef67fa540439ad331f13119e135: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
13:38:30.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.260100 17996 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
13:38:30.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.292237 18000 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 128c2ef67fa540439ad331f13119e135. Latest consensus state: current_term: 4 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:30.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.293015 18000 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.308991 18000 sys_catalog.cc:455] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.309754 18000 sys_catalog.cc:458] T 00000000000000000000000000000000 P 84ff1fb2b572460e9fe8d2efeb514e46 [sys.catalog]: This master's current role is: FOLLOWER
13:38:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.321722 17996 sys_catalog.cc:455] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "128c2ef67fa540439ad331f13119e135" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b20ae5c389e34802834399d6011ed84f" member_type: VOTER last_known_addr { host: "127.16.120.190" port: 39275 } } peers { permanent_uuid: "84ff1fb2b572460e9fe8d2efeb514e46" member_type: VOTER last_known_addr { host: "127.16.120.189" port: 38937 } } peers { permanent_uuid: "128c2ef67fa540439ad331f13119e135" member_type: VOTER last_known_addr { host: "127.16.120.188" port: 34525 } } }
13:38:30.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.322475 17996 sys_catalog.cc:458] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 [sys.catalog]: This master's current role is: LEADER
13:38:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:30.333117 17821 consensus_peers.cc:485] T 00000000000000000000000000000000 P 128c2ef67fa540439ad331f13119e135 -> Peer b20ae5c389e34802834399d6011ed84f (127.16.120.190:39275): Couldn't send request to peer b20ae5c389e34802834399d6011ed84f. Status: Network error: Client connection negotiation failed: client connection to 127.16.120.190:39275: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
13:38:30.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:29.159277 17964 server_base.cc:1034] running on GCE node
13:38:30.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.159566 17989 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:30.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:29.156239 17986 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:38:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.545866 17964 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:38:30.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.556066 17964 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:38:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.559695 17964 hybrid_clock.cc:648] HybridClock initialized: now 1714138710559657 us; error 44 us; skew 500 ppm
13:38:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.560758 17964 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:38:30.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:38:30 dist-test-slave-6qgw krb5kdc[16877](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138710, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.120.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:38:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.570163 17964 init.cc:377] Logged in from keytab as kudu/127.16.120.190@KRBTEST.COM (short username kudu)
13:38:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.584888 17964 webserver.cc:460] Webserver started at http://127.16.120.190:35425/ using document root <none> and password file <none>
13:38:30.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.586042 17964 fs_manager.cc:359] Metadata directory not provided
13:38:30.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.586292 17964 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:38:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.596410 17964 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
13:38:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.602192 18010 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:38:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.603528 17964 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.603845 17964 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data,/tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b20ae5c389e34802834399d6011ed84f"
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:37:59 on dist-test-slave-6qgw"
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.607067 17964 fs_report.cc:383] FS layout report
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/wal
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/data
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:38:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:38:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:38:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:38:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:38:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:38:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:38:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:38:30.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.718391 17964 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:38:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.725637 17964 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:38:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.726154 17964 kserver.cc:163] Server-wide thread pool size limit: 3276
13:38:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.958986 17964 rpc_server.cc:307] RPC server started. Bound to: 127.16.120.190:39275
13:38:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.959102 18066 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.120.190:39275 every 8 connection(s)
13:38:30.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.961921 17964 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbfnQDJ/test-tmp/mini-kudu-cluster2748278236480635741/master-0/data/info.pb
13:38:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:30.970039 16866 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu as pid 17964
13:38:30.971 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:38:30.971 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.120.188:34525(127.16.120.188:34525)] connecting to peer
13:38:30.976 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.120.188:34525(127.16.120.188:34525)] Successfully connected to peer
13:38:31.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:31.015108 18067 sys_catalog.cc:263] Verifying existing consensus state
13:38:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:31.023073 18067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Bootstrap starting.
13:38:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:31.023103 18069 negotiation.cc:344] Unauthorized connection attempt: Server connection negotiation failed: server connection from 127.0.0.1:33988: token expired
13:38:31.025 [DEBUG - kudu-nio-0] (Negotiator.java:352) peer /127.16.120.188:34525 sent connection negotiation error: Not authorized: token expired
13:38:31.028 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.120.188:34525(127.16.120.188:34525)] handling channelInactive
13:38:31.028 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.120.188:34525(127.16.120.188:34525)] cleaning up while in state NEGOTIATION_FAILED due to: connection closed
13:38:31.030 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.16.120.188:34525(127.16.120.188:34525)] connecting to peer
13:38:31.035 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.120.188:34525(127.16.120.188:34525)] Successfully connected to peer
13:38:31.037 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.120.188:34525(127.16.120.188:34525)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
13:38:31.037 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.120.188:34525(127.16.120.188:34525)] handling channelInactive
13:38:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:31.039489 18069 negotiation.cc:337] Failed RPC negotiation. Trace:
13:38:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.032321 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:33992 (local address 127.16.120.188:34525)
13:38:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.034036 (+ 1715us) server_negotiation.cc:206] Beginning negotiation
13:38:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.034064 (+ 28us) server_negotiation.cc:399] Waiting for connection header
13:38:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.035628 (+ 1564us) server_negotiation.cc:407] Connection header received
13:38:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.035759 (+ 131us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
13:38:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.035796 (+ 37us) server_negotiation.cc:461] Received NEGOTIATE request from client
13:38:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.036135 (+ 339us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
13:38:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.036419 (+ 284us) server_negotiation.cc:226] Negotiated authn=SASL
13:38:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.038131 (+ 1712us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:33992: BlockingRecv error: recv got EOF from 127.0.0.1:33992 (error 108)
13:38:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1100}
13:38:31.040 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.120.188:34525: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
13:38:31.041 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.16.120.189:38937(127.16.120.189:38937)] connecting to peer
13:38:31.043 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.16.120.190:39275(127.16.120.190:39275)] connecting to peer
13:38:31.053 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.120.190:39275(127.16.120.190:39275)] Successfully connected to peer
13:38:31.064 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.120.189:38937(127.16.120.189:38937)] Successfully connected to peer
13:38:31.067 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.120.189:38937(127.16.120.189:38937)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
13:38:31.067 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.120.189:38937: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
13:38:31.067 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.120.189:38937(127.16.120.189:38937)] handling channelInactive
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:31.075249 18071 negotiation.cc:337] Failed RPC negotiation. Trace:
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.055772 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53836 (local address 127.16.120.189:38937)
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.063569 (+ 7797us) server_negotiation.cc:206] Beginning negotiation
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.063594 (+ 25us) server_negotiation.cc:399] Waiting for connection header
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.065553 (+ 1959us) server_negotiation.cc:407] Connection header received
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.065769 (+ 216us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.065807 (+ 38us) server_negotiation.cc:461] Received NEGOTIATE request from client
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.066285 (+ 478us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.066664 (+ 379us) server_negotiation.cc:226] Negotiated authn=SASL
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.073703 (+ 7039us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53836: BlockingRecv error: recv got EOF from 127.0.0.1:53836 (error 108)
13:38:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7215,"thread_start_us":6923,"threads_started":1}
13:38:31.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.120.190:39275(127.16.120.190:39275)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
13:38:31.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.120.190:39275: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
13:38:31.086 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.120.190:39275(127.16.120.190:39275)] handling channelInactive
13:38:31.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:31.085601 18067 log.cc:824] T 00000000000000000000000000000000 P b20ae5c389e34802834399d6011ed84f: Log is configured to *not* fsync() on all Append() calls
13:38:31.086 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.120.189:38937(127.16.120.189:38937)] disconnecting while in state TERMINATED
13:38:31.087 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.120.190:39275(127.16.120.190:39275)] disconnecting while in state TERMINATED
13:38:31.087 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.120.188:34525(127.16.120.188:34525)] disconnecting while in state TERMINATED
13:38:31.089 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:38:31.090 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:38:31.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:38:31.108126 18070 negotiation.cc:337] Failed RPC negotiation. Trace:
13:38:31.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.046519 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:57084 (local address 127.16.120.190:39275)
13:38:31.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.073571 (+ 27052us) server_negotiation.cc:206] Beginning negotiation
13:38:31.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.073614 (+ 43us) server_negotiation.cc:399] Waiting for connection header
13:38:31.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.073693 (+ 79us) server_negotiation.cc:407] Connection header received
13:38:31.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.082629 (+ 8936us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
13:38:31.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.082685 (+ 56us) server_negotiation.cc:461] Received NEGOTIATE request from client
13:38:31.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.084049 (+ 1364us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
13:38:31.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.090885 (+ 6836us) server_negotiation.cc:226] Negotiated authn=SASL
13:38:31.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0426 13:38:31.093123 (+ 2238us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57084: BlockingRecv error: recv got EOF from 127.0.0.1:57084 (error 108)
13:38:31.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":23981,"spinlock_wait_cycles":354560,"thread_start_us":24564,"threads_started":1}
13:38:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:31.118161 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17964
13:38:31.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:31.163995 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17872
13:38:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:31.190675 16866 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 17799
13:38:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:38:32Z chronyd exiting
13:38:33.324 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:38:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:33.325323 16142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 16421
13:38:33.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:33.350363 16142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 16582
13:38:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:33.375327 16142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 16725
13:38:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:33.400483 16142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 16154
13:38:33.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:33.432004 16142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 16224
13:38:33.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:38:33.459434 16142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbfnQDJ/build/tsan/bin/kudu with pid 16299
13:38:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:38:34Z chronyd exiting
13:38:35.527 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.16.120.188:34525,127.16.120.189:38937,127.16.120.190:39275). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158) ~[main/:?]
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.16.120.188:34525,127.16.120.189:38937,127.16.120.190:39275). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
13:38:35.794 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: giving up after 1 attempts
13:38:35.794 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.16.120.188:34525,127.16.120.189:38937,127.16.120.190:39275). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158) ~[main/:?]
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.16.120.188:34525,127.16.120.189:38937,127.16.120.190:39275). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more